1.33

add WebDav

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,22 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+Sponsors SeaweedFS via Patreon https://www.patreon.com/seaweedfs
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**System Setup**
+List the command line to start "weed master", "weed volume", "weed filer", "weed s3", "weed mount".
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

.gitignore

@@ -77,3 +77,6 @@ crashlytics-build.properties
 
 test_data
 build
+target
+*.class
+other/java/hdfs/dependency-reduced-pom.xml

.project

@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
-	<name>seaweedfs</name>
-	<comment></comment>
-	<projects>
-	</projects>
-	<buildSpec>
-	</buildSpec>
-	<natures>
-		<nature>goclipse.goNature</nature>
-		<nature>org.eclipse.wst.common.project.facet.core.nature</nature>
-	</natures>
-</projectDescription>

.travis.yml

@@ -1,18 +1,21 @@
 sudo: false
 language: go
 go:
-- 1.5
-- 1.6
-- tip
+  - 1.10.x
+  - 1.11.x
+  - 1.12.x
+  # - tip
 
 before_install:
-- export PATH=/home/travis/gopath/bin:$PATH
+  - export PATH=/home/travis/gopath/bin:$PATH
 
 install:
-- go get ./weed/...
+  - export CGO_ENABLED="0"
+  - go env
+  - go get -u ./weed/...
 
 script:
-- go test ./weed/...
+  - go test ./weed/...
 
 before_deploy:
   - make release
@@ -22,23 +25,26 @@ deploy:
   api_key:
     secure: ERL986+ncQ8lwAJUYDrQ8s2/FxF/cyNIwJIFCqspnWxQgGNNyokET9HapmlPSxjpFRF0q6L2WCg9OY3mSVRq4oI6hg1igOQ12KlLyN71XSJ3c8w0Ay5ho48TQ9l3f3Iu97mntBCe9l0R9pnT8wj1VI8YJxloXwUMG2yeTjA9aBI=
   file:
-  - build/linux_arm.tar.gz
-  - build/linux_arm64.tar.gz
-  - build/linux_386.tar.gz
-  - build/linux_amd64.tar.gz
-  - build/darwin_amd64.tar.gz
-  - build/windows_386.zip
-  - build/windows_amd64.zip
-  - build/freebsd_arm.tar.gz
-  - build/freebsd_amd64.tar.gz
-  - build/freebsd_386.tar.gz
-  - build/netbsd_arm.tar.gz
-  - build/netbsd_amd64.tar.gz
-  - build/netbsd_386.tar.gz
-  - build/openbsd_arm.tar.gz
-  - build/openbsd_amd64.tar.gz
-  - build/openbsd_386.tar.gz
+    - build/linux_arm.tar.gz
+    - build/linux_arm64.tar.gz
+    - build/linux_386.tar.gz
+    - build/linux_amd64.tar.gz
+    - build/linux_amd64_large_disk.tar.gz
+    - build/darwin_amd64.tar.gz
+    - build/darwin_amd64_large_disk.tar.gz
+    - build/windows_386.zip
+    - build/windows_amd64.zip
+    - build/windows_amd64_large_disk.zip
+    - build/freebsd_arm.tar.gz
+    - build/freebsd_amd64.tar.gz
+    - build/freebsd_386.tar.gz
+    - build/netbsd_arm.tar.gz
+    - build/netbsd_amd64.tar.gz
+    - build/netbsd_386.tar.gz
+    - build/openbsd_arm.tar.gz
+    - build/openbsd_amd64.tar.gz
+    - build/openbsd_386.tar.gz
   on:
     tags: true
     repo: chrislusf/seaweedfs
-    go: tip
+    go: 1.12.x

Dockerfile.go_build

@@ -1,6 +0,0 @@
-FROM cydev/go
-RUN go get github.com/chrislusf/seaweedfs/weed
-EXPOSE 8080
-EXPOSE 9333
-VOLUME /data
-ENTRYPOINT ["weed"]

Makefile

@@ -8,10 +8,13 @@ appname := weed
 
 sources := $(wildcard *.go)
 
-build = GOOS=$(1) GOARCH=$(2) go build -o build/$(appname)$(3) $(SOURCE_DIR)
+build = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -ldflags "-extldflags -static" -o build/$(appname)$(3) $(SOURCE_DIR)
 tar = cd build && tar -cvzf $(1)_$(2).tar.gz $(appname)$(3) && rm $(appname)$(3)
 zip = cd build && zip $(1)_$(2).zip $(appname)$(3) && rm $(appname)$(3)
 
+build_large = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -tags 5BytesOffset -ldflags "-extldflags -static" -o build/$(appname)$(3) $(SOURCE_DIR)
+tar_large = cd build && tar -cvzf $(1)_$(2)_large_disk.tar.gz $(appname)$(3) && rm $(appname)$(3)
+zip_large = cd build && zip $(1)_$(2)_large_disk.zip $(appname)$(3) && rm $(appname)$(3)
 
 all: build
 
@@ -32,9 +35,21 @@ linux: deps
 	mkdir -p linux
 	GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -o linux/$(BINARY) $(SOURCE_DIR)
 
-release: deps windows_build darwin_build linux_build bsd_build
+release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_darwin_build 5_byte_windows_build
 
 ##### LINUX BUILDS #####
+5_byte_linux_build:
+	$(call build_large,linux,amd64,)
+	$(call tar_large,linux,amd64)
+
+5_byte_darwin_build:
+	$(call build_large,darwin,amd64,)
+	$(call tar_large,darwin,amd64)
+
+5_byte_windows_build:
+	$(call build_large,windows,amd64,.exe)
+	$(call zip_large,windows,amd64,.exe)
+
 linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz
 
 build/linux_386.tar.gz: $(sources)

README.md

@@ -1,54 +1,138 @@
 # SeaweedFS
 
+
 [![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs)
 [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
-[download](https://github.com/chrislusf/seaweedfs/releases/latest)
+[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=604800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
 
 ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)
 
-[![](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=EEECLJ8QGTTPC) 
+<h2 align="center">Supporting SeaweedFS</h2>
 
-## Introduction
+SeaweedFS is an independent Apache-licensed open source project with its ongoing development made
+possible entirely thanks to the support of these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md).
+If you'd like to grow SeaweedFS even stronger, please consider joining our
+<a href="https://www.patreon.com/seaweedfs">sponsors on Patreon</a>.
+
+Platinum ($2500/month), Gold ($500/month): put your company logo on the SeaweedFS github page
+Generous Backer($50/month), Backer($10/month): put your name on the SeaweedFS backer page.
+
+Your support will be really appreciated by me and other supporters!
+
+<h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h3>
+
+<h4 align="center">Platinum</h4>
+
+<p align="center">
+  <a href="" target="_blank">
+    Add your name or icon here
+  </a>
+</p>
+
+<h4 align="center">Gold</h4>
+
+<table>
+  <tbody>
+    <tr>
+      <td align="center" valign="middle">
+        <a href="" target="_blank">
+          Add your name or icon here
+        </a>
+      </td>
+    </tr>
+    <tr></tr>
+  </tbody>
+</table>
+
+---
+
+
+- [Download Binaries for different platforms](https://github.com/chrislusf/seaweedfs/releases/latest)
+- [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTc4MmVlYmFlNjBmZTgzZmJlYmI1MDE1YzkyNWYyZjkwZDFiM2RlMDdjNjVlNjdjYzc4NGFhZGIyYzEyMzJkYTA)
+- [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
+- [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
+- [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
+
+Table of Contents
+=================
+
+* [Introduction](#introduction)
+* [Features](#features)
+    * [Additional Features](#additional-features)
+    * [Filer Features](#filer-features)
+* [Example Usage](#example-usage)
+* [Architecture](#architecture)
+* [Compared to Other File Systems](#compared-to-other-file-systems)
+    * [Compared to HDFS](#compared-to-hdfs)
+    * [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
+    * [Compared to GlusterFS](#compared-to-glusterfs)
+    * [Compared to Ceph](#compared-to-ceph)
+* [Dev Plan](#dev-plan)
+* [Installation Guide](#installation-guide)
+* [Disk Related Topics](#disk-related-topics)
+* [Benchmark](#Benchmark)
+* [License](#license)
+
+## Introduction ##
 
 SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
 
 1. to store billions of files!
 2. to serve the files fast!
 
-Instead of supporting full POSIX file system semantics, SeaweedFS choose to implement only a key->file mapping. Similar to "NoSQL", you might call it "NoFS".
+SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages file volumes, and it lets these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (just one disk read operation).
 
-Instead of managing all file metadata in a central master, the central master only manages file volumes, and it lets these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (just one disk read operation).
+There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
 
-There is only a 40 bytes disk storage overhead for each file's metadata. It is so simple with O(1) disk read that you are welcome to challenge the performance with your actual use cases.
+SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf).
 
-SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). SeaweedFS is currently growing, with more features on the way.
+SeaweedFS can work very well with just the object store. [[Filer]] can then be added later to support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql/Postgres/Redis/Cassandra/LevelDB.
 
-[Download latest compiled binaries for different platforms](https://github.com/chrislusf/seaweedfs/releases/latest)
+[Back to TOC](#table-of-contents)
 
-[SeaweedFS Discussion Group](https://groups.google.com/d/forum/seaweedfs)
+## Features ##
 
-## Additional Features
-* Can choose no replication or different replication level, rack and data center aware
-* Automatic master servers failover - no single point of failure (SPOF)
-* Automatic Gzip compression depending on file mime type
-* Automatic compaction to reclaimed disk spaces after deletion or update
+[Back to TOC](#table-of-contents)
+
+## Additional Features ##
+* Can choose no replication or different replication levels, rack and data center aware.
+* Automatic master servers failover - no single point of failure (SPOF).
+* Automatic Gzip compression depending on file mime type.
+* Automatic compaction to reclaim disk space after deletion or update.
 * Servers in the same cluster can have different disk spaces, file systems, OS etc.
-* Adding/Removing servers does **not** cause any data re-balancing
-* Optional [filer server][Filer] provides "normal" directories and files via http
-* Optionally fix the orientation for jpeg pictures
-* Support Etag, Accept-Range, Last-Modified, etc.
-* Support in-memory/leveldb/boltdb mode tuning for memory/performance balance.
+* Adding/Removing servers does **not** cause any data re-balancing.
+* Optionally fix the orientation for jpeg pictures.
+* Support ETag, Accept-Range, Last-Modified, etc.
+* Support in-memory/leveldb/boltdb/btree mode tuning for memory/performance balance.
 
-[Filer]: https://github.com/chrislusf/seaweedfs/wiki/Filer
+[Back to TOC](#table-of-contents)
 
-## Example Usage
-By default, the master node runs on port 9333, and the volume nodes runs on port 8080.
-Here I will start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. I just use localhost as an example.
+## Filer Features ##
+* [filer server][Filer] provide "normal" directories and files via http.
+* [mount filer][Mount] to read and write files directly as a local directory via FUSE.
+* [Amazon S3 compatible API][AmazonS3API] to access files with S3 tooling.
+* [Hadoop Compatible File System][Hadoop] to access files from Hadoop/Spark/Flink/etc jobs.
+* [Async Backup To Cloud][BackupToCloud] has extremely fast local access and backups to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
+* [WebDAV] access as a mapped drive on Mac and Windows, or from mobile devices.
 
-SeaweedFS uses HTTP REST operations to write, read, delete. The responses are in JSON or JSONP format.
+[Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files
+[Mount]: https://github.com/chrislusf/seaweedfs/wiki/Mount
+[AmazonS3API]: https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API
+[BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Backup-to-Cloud
+[Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
+[WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV
 
-### Start Master Server
+[Back to TOC](#table-of-contents)
+
+## Example Usage ##
+
+By default, the master node runs on port 9333, and the volume nodes run on port 8080.
+Let's start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. We'll use localhost as an example.
+
+SeaweedFS uses HTTP REST operations to read, write, and delete. The responses are in JSON or JSONP format.
+
+### Start Master Server ###
 
 ```
 > ./weed master
@@ -61,56 +145,56 @@ SeaweedFS uses HTTP REST operations to write, read, delete. The responses are in
 > weed volume -dir="/tmp/data2" -max=10 -mserver="localhost:9333" -port=8081 &
 ```
 
-
 ### Write File ###
 
 To upload a file: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server url:
 
 ```
-> curl -X POST http://localhost:9333/dir/assign
+> curl http://localhost:9333/dir/assign
 {"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
 ```
 
-Second, to store the file content, send a HTTP multi-part PUT or POST request to `url + '/' + fid` from the response:
+Second, to store the file content, send a HTTP multi-part POST request to `url + '/' + fid` from the response:
 
 ```
-> curl -X PUT -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6
-{"size": 43234}
+> curl -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6
+{"name":"myphoto.jpg","size":43234,"eTag":"1cc0118e"}
 ```
 
-To update, send another PUT or POST request with updated file content.
+To update, send another POST request with updated file content.
 
 For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
 
 ```
 > curl -X DELETE http://127.0.0.1:8080/3,01637037d6
 ```
+
 ### Save File Id ###
 
-Now you can save the `fid`, 3,01637037d6 in this case, to a database field.
+Now, you can save the `fid`, 3,01637037d6 in this case, to a database field.
 
 The number 3 at the start represents a volume id. After the comma, it's one file key, 01, and a file cookie, 637037d6.
 
 The volume id is an unsigned 32-bit integer. The file key is an unsigned 64-bit integer. The file cookie is an unsigned 32-bit integer, used to prevent URL guessing.
 
-The file key and file cookie are both coded in hex. You can store the <volume id, file key, file cookie> tuple in your own format, or simply store the `fid` as string.
+The file key and file cookie are both coded in hex. You can store the <volume id, file key, file cookie> tuple in your own format, or simply store the `fid` as a string.
 
-If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most usage would not need 2^32 volumes.
+If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most uses will not need 2^32 volumes.
 
-If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, 4-byte integer for file cookie. So 16 bytes are more than enough.
+If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, and a 4-byte integer for the file cookie. So 16 bytes are more than enough.
 
 ### Read File ###
 
 Here is an example of how to render the URL.
 
-First lookup the volume server's URLs by the file's volumeId:
+First look up the volume server's URLs by the file's volumeId:
 
 ```
 > curl http://localhost:9333/dir/lookup?volumeId=3
-{"locations":[{"publicUrl":"localhost:8080","url":"localhost:8080"}]}
+{"volumeId":"3","locations":[{"publicUrl":"localhost:8080","url":"localhost:8080"}]}
 ```
 
-Since (usually) there are not too many volume servers, and volumes don't move often, you can cache the results most of the time. Depends on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read.)
+Since (usually) there are not too many volume servers, and volumes don't move often, you can cache the results most of the time. Depending on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read.
 
 Now you can take the public url, render the url or directly read from the volume server via url:
 
@@ -130,10 +214,12 @@ If you want a nicer URL, you can use one of these alternative URL formats:
  http://localhost:8080/3,01637037d6
 ```
 
-If you want get a scaled version of an image, you can add some params:
+If you want to get a scaled version of an image, you can add some params:
 
 ```
 http://localhost:8080/3/01637037d6.jpg?height=200&width=200
+http://localhost:8080/3/01637037d6.jpg?height=200&width=200&mode=fit
+http://localhost:8080/3/01637037d6.jpg?height=200&width=200&mode=fill
 ```
 
 ### Rack-Aware and Data Center-Aware Replication ###
@@ -141,7 +227,7 @@ http://localhost:8080/3/01637037d6.jpg?height=200&width=200
 SeaweedFS applies the replication strategy at a volume level. So, when you are getting a file id, you can specify the replication strategy. For example:
 
 ```
-curl -X POST http://localhost:9333/dir/assign?replication=001
+curl http://localhost:9333/dir/assign?replication=001
 ```
 
 The replication parameter options are:
@@ -161,9 +247,9 @@ More details about replication can be found [on the wiki][Replication].
 
 You can also set the default replication strategy when starting the master server.
 
-### Allocate File Key on specific data center ###
+### Allocate File Key on Specific Data Center ###
 
-Volume servers can start with a specific data center name:
+Volume servers can be started with a specific data center name:
 
 ```
  weed volume -dir=/tmp/1 -port=8080 -dataCenter=dc1
@@ -187,13 +273,15 @@ When requesting a file key, an optional "dataCenter" parameter can limit the ass
 [feat-3]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#upload-large-files
 [feat-4]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#collection-as-a-simple-name-space
 
+[Back to TOC](#table-of-contents)
+
 ## Architecture ##
 
 Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.
 
-The main drawback is that the central master can't handle many small files efficiently, and since all read requests need to go through the chunk master, might not scale well for many concurrent users.
+The main drawback is that the central master can't handle many small files efficiently, and since all read requests need to go through the chunk master, so it might not scale well for many concurrent users.
 
-Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is size 32GB, and can hold a lot of files. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is fairly small amount of data and is generally stable.
+Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is 32GB in size, and can hold a lot of files. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is a fairly small amount of data and is generally stable.
 
 The actual file metadata is stored in each volume on volume servers. Since each volume server only manages metadata of files on its own disk, with only 16 bytes for each file, all file access can read file metadata just from memory and only needs one disk operation to actually read file data.
 
@@ -203,7 +291,7 @@ For comparison, consider that an xfs inode structure in Linux is 536 bytes.
 
 The architecture is fairly simple. The actual data is stored in volumes on storage nodes. One volume server can have multiple volumes, and can both support read and write access with basic authentication.
 
-All volumes are managed by a master server. The master server contains volume id to volume server mapping. This is fairly static information, and could be cached easily.
+All volumes are managed by a master server. The master server contains the volume id to volume server mapping. This is fairly static information, and can be easily cached.
 
 On each write request, the master server also generates a file key, which is a growing 64-bit unsigned integer. Since write requests are not generally as frequent as read requests, one master server should be able to handle the concurrency well.
 
@@ -211,111 +299,149 @@ On each write request, the master server also generates a file key, which is a g
 
 When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node url) for the file. The client then contacts the volume node and POSTs the file content.
 
-When a client needs to read a file based on (volume id, file key, file cookie), it can ask the master server by the volume id for the (volume node url, volume node public url), or retrieve this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
+When a client needs to read a file based on (volume id, file key, file cookie), it asks the master server by the volume id for the (volume node url, volume node public url), or retrieves this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
 
 Please see the example for details on the write-read process.
 
 ### Storage Size ###
 
-In the current implementation, each volume can be 8x2^32 bytes (32GiB). This is because of we align content to 8 bytes. We can easily increase this to 64G, or 128G, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
+In the current implementation, each volume can hold 32 gibibytes (32GiB or 8x2^32 bytes). This is because we align content to 8 bytes. We can easily increase this to 64GiB, or 128GiB, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
 
-There can be 2^32 volumes. So total system size is 8 x 2^32 bytes x 2^32 = 8 x 4GiB x 4Gi = 128EiB (2^67 bytes, or 128 exbibytes).
+There can be 4 gibibytes (4GiB or 2^32 bytes) of volumes. So the total system size is 8 x 4GiB x 4GiB which is 128 exbibytes (128EiB or 2^67 bytes).
 
 Each individual file size is limited to the volume size.
 
 ### Saving memory ###
 
-All file meta information on volume server is readable from memory without disk access. Each file just takes an 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own the space cost for the map. But usually the disk runs out before the memory does.
+All file meta information stored on an volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
 
+[Back to TOC](#table-of-contents)
 
 ## Compared to Other File Systems ##
 
 Most other distributed file systems seem more complicated than necessary.
 
-### Compared to Ceph ###
+SeaweedFS is meant to be fast and simple, in both setup and operation. If you do not understand how it works when you reach here, we've failed! Please raise an issue with any questions or update this file with clarifications.
 
-Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/chrislusf/seaweedfs/issues/120)
-
-SeaweedFS is meant to be fast and simple, both during usage and during setup. If you do not understand how it works when you reach here, we failed! Please raise an issue with any questions or update this file with clarifications.
-
-SeaweedFS has a centralized master to lookup free volumes, while Ceph uses hashing to locate its objects. Having a centralized master makes it easy to code and manage. HDFS/GFS has the single name node for years. SeaweedFS now support multiple master nodes.
-
-Ceph hashing avoids SPOF, but makes it complicated when moving or adding servers.
+[Back to TOC](#table-of-contents)
 
 ### Compared to HDFS ###
 
-HDFS uses the chunk approach for each file, and is ideal for streaming large files.
+HDFS uses the chunk approach for each file, and is ideal for storing large files.
 
 SeaweedFS is ideal for serving relatively smaller files quickly and concurrently.
 
 SeaweedFS can also store extra large files by splitting them into manageable data chunks, and store the file ids of the data chunks into a meta chunk. This is managed by "weed upload/download" tool, and the weed master or volume servers are agnostic about it.
 
-### Compared to MogileFS ###
+[Back to TOC](#table-of-contents)
 
-SeaweedFS has 2 components: directory server, storage nodes.
+### Compared to GlusterFS, Ceph ###
 
-MogileFS has 3 components: tracers, database, storage nodes.
+The architectures are mostly the same. SeaweedFS aims to store and read files fast, with a simple and flat architecture. The main differences are
 
-One more layer means slower access, more operation complexity, more failure possibility.
+* SeaweedFS optimizes for small files, ensuring O(1) disk seek operation, and can also handle large files.
+* SeaweedFS statically assigns a volume id for a file. Locating file content becomes just a lookup of the volume id, which can be easily cached.
+* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Cassandra, Redis, MySql, Postgres, etc, and is easy to customized.
+* SeaweedFS Volume server also communicates directly with clients via HTTP, supporting range queries, direct uploads, etc.
+
+| System         | File Meta                       | File Content Read| POSIX  | REST API | Optimized for small files |
+| -------------  | ------------------------------- | ---------------- | ------ | -------- | ------------------------- |
+| SeaweedFS      | lookup volume id, cacheable     | O(1) disk seek   |        | Yes      | Yes                       |
+| SeaweedFS Filer| Linearly Scalable, Customizable | O(1) disk seek   | FUSE   | Yes      | Yes                       |
+| GlusterFS      | hashing          |                  | FUSE, NFS          |          |                           |
+| Ceph           | hashing + rules  |                  | FUSE               | Yes      |                           |
+
+[Back to TOC](#table-of-contents)
 
 ### Compared to GlusterFS ###
 
-SeaweedFS is not POSIX compliant, and has a simple implementation.
+GlusterFS stores files, both directories and content, in configurable volumes called "bricks".
 
-GlusterFS is POSIX compliant, and is much more complex.
+GlusterFS hashes the path and filename into ids, and assigned to virtual volumes, and then mapped to "bricks".
 
-### Compared to MongoDB's GridFS ###
+[Back to TOC](#table-of-contents)
 
-Mongo's GridFS splits files into chunks and manage chunks in the central MongoDB. For every read or write request, the database needs to query the metadata. It's OK if this is not yet a bottleneck, but for a lot of concurrent reads this unnecessary query could slow things down.
+### Compared to Ceph ###
 
-Since files are chunked(default to 256KB), there will be multiple metadata readings and multiple chunk readings, linear to the file size. One 2.56MB file would require at least 20 disk read requests.
+Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/chrislusf/seaweedfs/issues/120)
 
-On the contrary, SeaweedFS uses large file volume of 32G size to store lots of files, and only manages file volumes in the master server. Each volume manages file metadata itself. So all file metadata is spread across the volume nodes, and just one disk read is needed.
+SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage.
 
-## Dev plan ##
+Same as SeaweedFS, Ceph is also based on the object store RADOS. Ceph is rather complicated with mixed reviews.
+
+Ceph uses CRUSH hashing to automatically manage the data placement. SeaweedFS places data by assigned volumes.
+
+SeaweedFS is optimized for small files. Small files are stored as one continuous block of content, with at most 8 unused bytes between files. Small file access is O(1) disk read.
+
+SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Redis, Cassandra, to manage file directories. There are proven, scalable, and easier to manage.
+
+| SeaweedFS         | comparable to Ceph | advantage |
+| -------------  | ------------- | ---------------- |
+| Master  | MDS | simpler |
+| Volume  | OSD | optimized for small files |
+| Filer  | Ceph FS | linearly scalable, Customizable, O(1) or O(logN) |
+
+[Back to TOC](#table-of-contents)
+
+## Dev Plan ##
 
 More tools and documentation, on how to maintain and scale the system. For example, how to move volumes, automatically balancing data, how to grow volumes, how to check system status, etc.
+Other key features include: Erasure Encoding, JWT security.
 
-This is a super exciting project! And I need helpers!
+This is a super exciting project! And we need helpers and [support](https://www.patreon.com/seaweedfs)!
 
+BTW, We suggest run the code style check script `util/gostd` before you push your branch to remote, it will make SeaweedFS easy to review, maintain and develop:
 
-## Installation guide for users who are not familiar with golang
+```
+$ ./util/gostd
+```
 
-step 1: install go on your machine and setup the environment by following the instructions from the following link:
+[Back to TOC](#table-of-contents)
+
+## Installation Guide ##
+
+> Installation guide for users who are not familiar with golang
+
+Step 1: install go on your machine and setup the environment by following the instructions at:
 
 https://golang.org/doc/install
 
 make sure you set up your $GOPATH
 
 
-step 2: also you may need to install Mercurial by following the instructions below
+Step 2: also you may need to install Mercurial by following the instructions at:
 
 http://mercurial.selenic.com/downloads
 
+Step 3: download, compile, and install the project by executing the following command
 
-step 3: download, compile, and install the project by executing the following command
-
+```bash
 go get github.com/chrislusf/seaweedfs/weed
+```
 
-once this is done, you should see the executable "weed" under $GOPATH/bin
+Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
 
-step 4: after you modify your code locally, you could start a local build by calling "go install" under $GOPATH/src/github.com/chrislusf/seaweedfs/weed
+Step 4: after you modify your code locally, you could start a local build by calling `go install` under
 
-## Disk Related topics ##
+```
+$GOPATH/src/github.com/chrislusf/seaweedfs/weed
+```
+
+[Back to TOC](#table-of-contents)
+
+## Disk Related Topics ##
 
 ### Hard Drive Performance ###
 
-When testing read performance on SeaweedFS, it basically becomes performance test your hard drive's random read speed. Hard Drive usually get 100MB/s~200MB/s.
+When testing read performance on SeaweedFS, it basically becomes a performance test of your hard drive's random read speed. Hard drives usually get 100MB/s~200MB/s.
 
-### Solid State Disk
+### Solid State Disk ###
 
 To modify or delete small files, SSD must delete a whole block at a time, and move content in existing blocks to a new block. SSD is fast when brand new, but will get fragmented over time and you have to garbage collect, compacting blocks. SeaweedFS is friendly to SSD since it is append-only. Deletion and compaction are done on volume level in the background, not slowing reading and not causing fragmentation.
 
-## Not Planned
+[Back to TOC](#table-of-contents)
 
-POSIX support
-
-## Benchmark
+## Benchmark ##
 
 My Own Unscientific Single Machine Results on Mac Book with Solid State Disk, CPU: 1 Intel Core i7 2.6GHz.
 
@@ -368,8 +494,9 @@ Percentage of the requests served within a certain time (ms)
   100%     20.7 ms
 ```
 
+[Back to TOC](#table-of-contents)
 
-## License
+## License ##
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -382,3 +509,9 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+
+[Back to TOC](#table-of-contents)
+
+## Stargazers over time ##
+
+[![Stargazers over time](https://starcharts.herokuapp.com/chrislusf/seaweedfs.svg)](https://starcharts.herokuapp.com/chrislusf/seaweedfs)

backers.md

@@ -0,0 +1,12 @@
+<h1 align="center">Sponsors &amp; Backers</h1>
+
+- [Become a backer or sponsor on Patreon](https://www.patreon.com/seaweedfs).
+
+<h2 align="center">Generous Backers ($50+)</h2>
+
+- [Evercam Camera Management Software](https://evercam.io/)
+
+<h2 align="center">Backers</h2>
+
+- [ColorfulClouds Tech Co. Ltd.](https://caiyunai.com/)
+- [Haravan - Ecommerce Platform](https://www.haravan.com)

docker/Dockerfile

@@ -1,16 +1,43 @@
-FROM frolvlad/alpine-glibc:alpine-3.4
+FROM frolvlad/alpine-glibc
 
-RUN apk add --no-cache --virtual=build-dependencies --update curl wget ca-certificates && \
-    wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o '/chrislusf/seaweedfs/releases/download/.*/linux_amd64.tar.gz') && \
+# Supercronic install settings
+ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.8/supercronic-linux-amd64 \
+    SUPERCRONIC=supercronic-linux-amd64 \
+    SUPERCRONIC_SHA1SUM=be43e64c45acd6ec4fce5831e03759c89676a0ea
+
+# Install SeaweedFS and Supercronic ( for cron job mode )
+# Tried to use curl only (curl -o /tmp/linux_amd64.tar.gz ...), however it turned out that the following tar command failed with "gzip: stdin: not in gzip format"
+RUN apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
+    wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o 'chrislusf/seaweedfs/releases/download/.*/linux_amd64.tar.gz') && \
     tar -C /usr/bin/ -xzvf /tmp/linux_amd64.tar.gz && \
-    apk del curl wget ca-certificates build-dependencies && \
+    curl -fsSLO "$SUPERCRONIC_URL" && \
+    echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
+    chmod +x "$SUPERCRONIC" && \
+    mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
+    ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
+    apk del build-dependencies && \
     rm -rf /tmp/*
 
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
 EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
 EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+
+RUN mkdir -p /data/filerdb
 
 VOLUME /data
 
+COPY filer.toml /etc/seaweedfs/filer.toml
 COPY entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 

docker/Dockerfile.go_build

@@ -0,0 +1,29 @@
+FROM golang:latest
+RUN go get github.com/chrislusf/seaweedfs/weed
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+
+RUN mkdir -p /data/filerdb
+
+VOLUME /data
+
+RUN mkdir -p /etc/seaweedfs
+RUN cp /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
+RUN cp /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+RUN cp /go/bin/weed /usr/bin/
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/README.md

@@ -0,0 +1,21 @@
+# Docker
+
+
+## Try it out
+
+```bash
+
+wget https://raw.githubusercontent.com/chrislusf/seaweedfs/master/docker/seaweedfs-compose.yml
+
+docker-compose -f seaweedfs-compose.yml -p seaweedfs up
+
+```
+
+## Development
+
+```bash
+cd $GOPATH/src/github.com/chrislusf/seaweedfs/docker
+
+docker-compose -f dev-compose.yml -p seaweedfs up
+
+```

docker/dev-compose.yml

@@ -0,0 +1,43 @@
+version: '2'
+
+services:
+  master:
+    build:
+      context: .
+      dockerfile: Dockerfile.go_build
+    ports:
+    - 9333:9333
+    - 19333:19333
+    command: "master"
+  volume:
+    build:
+      context: .
+      dockerfile: Dockerfile.go_build
+    ports:
+    - 8080:8080
+    - 18080:18080
+    command: 'volume -max=5 -mserver="master:9333" -port=8080'
+    depends_on:
+    - master
+  filer:
+    build:
+      context: .
+      dockerfile: Dockerfile.go_build
+    ports:
+    - 8888:8888
+    - 18888:18888
+    command: 'filer -master="master:9333"'
+    depends_on:
+    - master
+    - volume
+  s3:
+    build:
+      context: .
+      dockerfile: Dockerfile.go_build
+    ports:
+    - 8333:8333
+    command: 's3 -filer="filer:8888"'
+    depends_on:
+    - master
+    - volume
+    - filer

docker/docker-compose.yml

@@ -1,23 +0,0 @@
-version: '2'
-
-services:
-  master:
-    image: chrislusf/seaweedfs
-    ports:
-      - 9333:9333
-    command: "master"
-    networks:
-      default:
-        aliases:
-          - seaweed_master
-  volume:
-    image: chrislusf/seaweedfs
-    ports:
-      - 8080:8080
-    command: 'volume -max=5 -mserver="master:9333" -port=8080'
-    depends_on: 
-     - master
-    networks:
-      default:
-        aliases:
-          - seaweed_volume

docker/entrypoint.sh

@@ -28,6 +28,30 @@ case "$1" in
   	exec /usr/bin/weed $@ $ARGS
   	;;
 
+  'filer')
+  	ARGS="-ip `hostname -i` "
+  	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
+		ARGS="$ARGS -master=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
+	fi
+  	exec /usr/bin/weed $@ $ARGS
+	;;
+
+  's3')
+  	ARGS="-domainName=$S3_DOMAIN_NAME -key.file=$S3_KEY_FILE -cert.file=$S3_CERT_FILE"
+  	if [ -n "$FILER_PORT_8888_TCP_ADDR" ] ; then
+		ARGS="$ARGS -filer=$FILER_PORT_8888_TCP_ADDR:$FILER_PORT_8888_TCP_PORT"
+	fi
+  	exec /usr/bin/weed $@ $ARGS
+	;;
+
+  'cronjob')
+	MASTER=${WEED_MASTER-localhost:9333}
+	CRON_SCHEDULE=${CRON_SCHEDULE-*/5 * * * * *}
+	echo "$CRON_SCHEDULE" 'echo "volume.fix.replication" | weed shell -master='$MASTER > /crontab
+	echo "Running Crontab:"
+	cat /crontab
+	exec supercronic /crontab
+	;;
   *)
   	exec /usr/bin/weed $@
 	;;

docker/filer.toml

@@ -0,0 +1,3 @@
+[leveldb]
+enabled = true
+dir = "/data/filerdb"

docker/seaweedfs-compose.yml

@@ -0,0 +1,47 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+    - 9333:9333
+    - 19333:19333
+    command: "master"
+  volume:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+    - 8080:8080
+    - 18080:18080
+    command: 'volume -max=15 -mserver="master:9333" -port=8080'
+    depends_on:
+    - master
+  filer:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+    - 8888:8888
+    - 18888:18888
+    command: 'filer -master="master:9333"'
+    tty: true
+    stdin_open: true
+    depends_on:
+    - master
+    - volume
+  cronjob:
+    image: chrislusf/seaweedfs # use a remote image
+    command: 'cronjob'
+    environment:
+      # Run re-replication every 2 minutes
+      CRON_SCHEDULE: '*/2 * * * * *' # Default: '*/5 * * * * *'
+      WEED_MASTER: master:9333 # Default: localhost:9333
+    depends_on:
+    - master
+    - volume
+  s3:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+    - 8333:8333
+    command: 's3 -filer="filer:8888"'
+    depends_on:
+    - master
+    - volume
+    - filer

note/memory_usage.txt

@@ -1,278 +0,0 @@
-64 32G volumes consumes 10G memory
-Each volume has 25M index, so each cost 160MB memory
-
-
-
-Things happened when I use lots of threads ( almost 120 ) keeping read file from SeaweedFS.
-But I'm not so familiar with linux so I can't tell you exactly what happened.
-Next I'll show you things I know , if you need more info , contact me
-
-My SeaweedFS version is about 0.12
-
-1. top
-
-top - 12:07:37 up 1 day,  3:17,  2 users,  load average: 0.00, 0.00, 0.00
-Tasks: 152 total,   1 running, 151 sleeping,   0 stopped,   0 zombie
-Cpu(s):  0.0%us,  0.0%sy,  0.0%ni, 99.8%id,  0.1%wa,  0.0%hi,  0.0%si,  0.0%st
-Mem:  16269880k total, 16192364k used,    77516k free,    58172k buffers
-Swap:  2064376k total,    12324k used,  2052052k free,  2827520k cached
-
-  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
- 1499 root      20   0 11.6g  10g 1424 S  0.0 65.6   7:32.53 weedvolume
- 1498 root      20   0 3204m 2.1g 1428 S  0.0 13.5   4:36.59 weedvolume
- 1737 root      20   0 98868 4932 2920 S  0.0  0.0   0:00.56 sshd
- 1497 root      20   0  151m 4404 1152 S  0.0  0.0   1:21.40 weedmaster
- 1335 root      20   0 97816 3044 2896 S  0.0  0.0   0:00.76 sshd
-
-After system became steady ,  weedvolume used 65.6% memory .
-
-2. free -m
-
-             total       used       free     shared    buffers     cached
-Mem:         15888      15809         79          0         56       2758
--/+ buffers/cache:      12994       2894
-Swap:         2015         12       2003
-
-3. startup cmd
-
-screen -d -m /opt/weed/weedmaster -mdir /data/weeddata/ > /data/logs/weed/master.log &
-screen -d -m /opt/weed/weedvolume -volumes=0-64 -dir /data/weeddata/ -pulseSeconds=20 -publicUrl="x.y.z:9334" -port 9334 > /data/logs/weed/s01.log &
-screen -d -m /opt/weed/weedvolume -volumes=65-107 -dir /data/weeddata/ -pulseSeconds=20 -publicUrl="x.y.z:9335" -port 9335 > /data/logs/weed/s02.log &
-
-4. du -sh .
-
-32G     0.dat
-26M     0.idx
-8.2G    100.dat
-6.8M    100.idx
-8.2G    101.dat
-6.9M    101.idx
-8.2G    102.dat
-6.8M    102.idx
-8.2G    103.dat
-6.8M    103.idx
-8.2G    104.dat
-6.8M    104.idx
-8.2G    105.dat
-6.9M    105.idx
-8.2G    106.dat
-6.9M    106.idx
-8.2G    107.dat
-6.9M    107.idx
-32G     10.dat
-25M     10.idx
-32G     11.dat
-25M     11.idx
-32G     12.dat
-25M     12.idx
-32G     13.dat
-25M     13.idx
-32G     14.dat
-25M     14.idx
-32G     15.dat
-25M     15.idx
-32G     16.dat
-25M     16.idx
-32G     17.dat
-25M     17.idx
-32G     18.dat
-25M     18.idx
-32G     19.dat
-25M     19.idx
-32G     1.dat
-26M     1.idx
-32G     20.dat
-25M     20.idx
-32G     21.dat
-25M     21.idx
-32G     22.dat
-25M     22.idx
-32G     23.dat
-25M     23.idx
-32G     24.dat
-25M     24.idx
-32G     25.dat
-25M     25.idx
-32G     26.dat
-25M     26.idx
-32G     27.dat
-25M     27.idx
-32G     28.dat
-25M     28.idx
-32G     29.dat
-25M     29.idx
-32G     2.dat
-26M     2.idx
-32G     30.dat
-25M     30.idx
-32G     31.dat
-25M     31.idx
-32G     32.dat
-25M     32.idx
-32G     33.dat
-25M     33.idx
-32G     34.dat
-25M     34.idx
-32G     35.dat
-25M     35.idx
-32G     36.dat
-25M     36.idx
-32G     37.dat
-25M     37.idx
-32G     38.dat
-25M     38.idx
-32G     39.dat
-25M     39.idx
-32G     3.dat
-26M     3.idx
-32G     40.dat
-25M     40.idx
-32G     41.dat
-25M     41.idx
-32G     42.dat
-25M     42.idx
-32G     43.dat
-25M     43.idx
-32G     44.dat
-25M     44.idx
-32G     45.dat
-25M     45.idx
-32G     46.dat
-25M     46.idx
-32G     47.dat
-25M     47.idx
-32G     48.dat
-25M     48.idx
-32G     49.dat
-25M     49.idx
-32G     4.dat
-26M     4.idx
-32G     50.dat
-25M     50.idx
-32G     51.dat
-25M     51.idx
-32G     52.dat
-25M     52.idx
-32G     53.dat
-25M     53.idx
-32G     54.dat
-25M     54.idx
-32G     55.dat
-25M     55.idx
-32G     56.dat
-25M     56.idx
-32G     57.dat
-25M     57.idx
-32G     58.dat
-25M     58.idx
-32G     59.dat
-25M     59.idx
-32G     5.dat
-26M     5.idx
-32G     60.dat
-25M     60.idx
-32G     61.dat
-25M     61.idx
-32G     62.dat
-25M     62.idx
-32G     63.dat
-25M     63.idx
-32G     64.dat
-25M     64.idx
-8.2G    65.dat
-6.9M    65.idx
-8.2G    66.dat
-6.9M    66.idx
-8.2G    67.dat
-6.9M    67.idx
-8.2G    68.dat
-6.8M    68.idx
-8.2G    69.dat
-6.9M    69.idx
-32G     6.dat
-25M     6.idx
-8.2G    70.dat
-6.8M    70.idx
-8.2G    71.dat
-6.9M    71.idx
-8.2G    72.dat
-6.9M    72.idx
-8.2G    73.dat
-6.9M    73.idx
-8.2G    74.dat
-6.9M    74.idx
-8.2G    75.dat
-6.9M    75.idx
-8.1G    76.dat
-6.8M    76.idx
-8.2G    77.dat
-6.8M    77.idx
-8.2G    78.dat
-6.8M    78.idx
-8.1G    79.dat
-6.8M    79.idx
-32G     7.dat
-25M     7.idx
-8.2G    80.dat
-6.8M    80.idx
-8.2G    81.dat
-6.9M    81.idx
-8.2G    82.dat
-6.9M    82.idx
-8.2G    83.dat
-6.9M    83.idx
-8.2G    84.dat
-6.9M    84.idx
-8.2G    85.dat
-6.8M    85.idx
-8.2G    86.dat
-6.9M    86.idx
-8.2G    87.dat
-6.9M    87.idx
-8.2G    88.dat
-6.9M    88.idx
-8.2G    89.dat
-6.8M    89.idx
-32G     8.dat
-25M     8.idx
-8.2G    90.dat
-6.9M    90.idx
-8.1G    91.dat
-6.8M    91.idx
-8.1G    92.dat
-6.8M    92.idx
-8.1G    93.dat
-6.8M    93.idx
-8.2G    94.dat
-6.9M    94.idx
-8.2G    95.dat
-6.9M    95.idx
-8.2G    96.dat
-6.9M    96.idx
-8.2G    97.dat
-6.9M    97.idx
-8.2G    98.dat
-6.9M    98.idx
-8.2G    99.dat
-6.9M    99.idx
-32G     9.dat
-25M     9.idx
-4.0K    directory.seq
-
-You can see the volume 1-64 is now full.
-
-5. more log
-
-see logs.zip
-
-In messages you can see these lines: (Line 51095)
-
-Sep 26 06:14:31 wedb-01 kernel: auditd: page allocation failure. order:0, mode:0x20
-Sep 26 06:14:31 wedb-01 kernel: Pid: 1009, comm: auditd Not tainted 2.6.32-220.el6.x86_64 #1
-Sep 26 06:14:31 wedb-01 kernel: Call Trace:
-
-After those lines , the system deny any new network connect request
-
-6. /dir/status
-
-{"Machines":[{"Server":{"Url":"127.0.0.1:9335","PublicUrl":"x.y.z:9335"},"Volumes":[{"Id":106,"Size":8728909632},{"Id":66,"Size":8729852744},{"Id":90,"Size":8747834896},{"Id":103,"Size":8718106024},{"Id":87,"Size":8732133512},{"Id":96,"Size":8737251904},{"Id":80,"Size":8704130712},{"Id":77,"Size":8717989496},{"Id":70,"Size":8731474744},{"Id":94,"Size":8758656144},{"Id":107,"Size":8729599232},{"Id":67,"Size":8736848088},{"Id":91,"Size":8665847760},{"Id":100,"Size":8703272552},{"Id":84,"Size":8745121528},{"Id":97,"Size":8713031744},{"Id":81,"Size":8726088872},{"Id":74,"Size":8738588152},{"Id":71,"Size":8729349920},{"Id":95,"Size":8741526896},{"Id":104,"Size":8699374736},{"Id":88,"Size":8740362880},{"Id":101,"Size":8711832992},{"Id":85,"Size":8723479552},{"Id":78,"Size":8700345400},{"Id":75,"Size":8727796912},{"Id":68,"Size":8698607440},{"Id":92,"Size":8682683056},{"Id":105,"Size":8741226152},{"Id":65,"Size":8725365752},{"Id":89,"Size":8703062600},{"Id":98,"Size":8742331560},{"Id":82,"Size":8762554952},{"Id":79,"Size":8696300376},{"Id":72,"Size":8708217304},{"Id":69,"Size":8740268144},{"Id":93,"Size":8685060320},{"Id":102,"Size":8708695352},{"Id":86,"Size":8783247776},{"Id":99,"Size":8753463608},{"Id":83,"Size":8725963952},{"Id":76,"Size":8694693536},{"Id":73,"Size":8733560832}]},{"Server":{"Url":"127.0.0.1:9334","PublicUrl":"x.y.z:9334"},"Volumes":[{"Id":34,"Size":33415706800},{"Id":58,"Size":33569224784},{"Id":18,"Size":33474649968},{"Id":55,"Size":33542422680},{"Id":15,"Size":33517247576},{"Id":48,"Size":33574860328},{"Id":8,"Size":33511257144},{"Id":45,"Size":33463948408},{"Id":5,"Size":34317702920},{"Id":29,"Size":33465695776},{"Id":38,"Size":33553119624},{"Id":62,"Size":33448316736},{"Id":22,"Size":33566586296},{"Id":35,"Size":33493733728},{"Id":59,"Size":33498554904},{"Id":19,"Size":33493313784},{"Id":52,"Size":33552978448},{"Id":12,"Size":33505183752},{"Id":49,"Size":33603029896},{"Id":9,"Size":33515778064},{"Id":42,"Size":33500402248},{"Id":2,"Size":34223232992},{"Id":26,"Size":33526519600},{"Id":39,"Size":33580414336},{"Id":63,"Size":33476332456},{"Id":23,"Size":33543872592},{"Id":32,"Size":33515290168},{"Id":56,"Size":33499171184},{"Id":16,"Size":33556591168},{"Id":64,"Size":33495148616},{"Id":53,"Size":33467738560},{"Id":13,"Size":33596873960},{"Id":46,"Size":33508120448},{"Id":6,"Size":33417470256},{"Id":30,"Size":33532933992},{"Id":43,"Size":33591802008},{"Id":3,"Size":34270682080},{"Id":27,"Size":33525736944},{"Id":36,"Size":33443597824},{"Id":60,"Size":33427931336},{"Id":20,"Size":33499083096},{"Id":33,"Size":33531396280},{"Id":57,"Size":33578015104},{"Id":17,"Size":33510525480},{"Id":50,"Size":33503123704},{"Id":10,"Size":33502391608},{"Id":47,"Size":33521868568},{"Id":7,"Size":33497101664},{"Id":31,"Size":33426905232},{"Id":40,"Size":33472978696},{"Id":0,"Size":34337344304},{"Id":24,"Size":33550157192},{"Id":37,"Size":33477162720},{"Id":61,"Size":33537175080},{"Id":21,"Size":33517192456},{"Id":54,"Size":33480720288},{"Id":14,"Size":33513192896},{"Id":51,"Size":33531336080},{"Id":11,"Size":33562385088},{"Id":44,"Size":33554479104},{"Id":4,"Size":34333127520},{"Id":28,"Size":33510503000},{"Id":41,"Size":33574922928},{"Id":1,"Size":34307181368},{"Id":25,"Size":33542834568}]}],"Writers":[106,66,90,103,87,96,80,77,70,94,107,67,91,100,84,97,81,74,71,95,104,88,101,85,78,75,68,92,105,65,89,98,82,79,72,69,93,102,86,99,83,76,73,34,58,18,55,15,48,8,45,5,29,38,62,22,35,59,19,52,12,49,9,42,2,26,39,63,23,32,56,16,64,53,13,46,6,30,43,3,27,36,60,20,33,57,17,50,10,47,7,31,40,0,24,37,61,21,54,14,51,11,44,4,28,41,1,25],"FileIdSequence":110250000}

note/replication.txt

@@ -1,109 +0,0 @@
-1. each file can choose the replication factor
-2. replication granularity is in volume level
-3. if not enough spaces, we can automatically decrease some volume's the replication factor, especially for cold data
-4. plan to support migrating data to cheaper storage
-5. plan to manual volume placement, access-based volume placement, auction based volume placement
-
-When a new volume server is started, it reports 
-  1. how many volumes it can hold
-  2. current list of existing volumes and each volume's replication type
-Each volume server remembers:
-  1. current volume ids
-  2. replica locations are read from the master
-
-The master assign volume ids based on
-  1. replication factor
-     data center, rack
-  2. concurrent write support
-On master, stores the replication configuration
-{
-  replication:{
-    {type:"00", min_volume_count:3, weight:10},
-    {type:"01", min_volume_count:2, weight:20},
-    {type:"10", min_volume_count:2, weight:20},
-    {type:"11", min_volume_count:3, weight:30},
-    {type:"20", min_volume_count:2, weight:20}
-  },
-  port:9333,
-}
-Or manually via command line
-  1. add volume with specified replication factor
-  2. add volume with specified volume id
-
-
-If duplicated volume ids are reported from different volume servers, 
-the master determines the replication factor of the volume,
-if less than the replication factor, the volume is in readonly mode
-if more than the replication factor, the volume will purge the smallest/oldest volume
-if equal, the volume will function as usual
-
-
-Use cases:
-  on volume server
-    1. weed volume -mserver="xx.xx.xx.xx:9333" -publicUrl="good.com:8080" -dir="/tmp" -volumes=50
-  on weed master
-    1. weed master -port=9333
-      generate a default json configuration file if doesn't exist
-      
-Bootstrap
-  1. at the very beginning, the system has no volumes at all.
-When data node starts:
-  1. each data node send to master its existing volumes and max volume blocks
-  2. master remembers the topology/data_center/rack/data_node/volumes
-     for each replication level, stores
-       volume id ~ data node
-       writable volume ids
-If any "assign" request comes in
-  1. find a writable volume with the right replicationLevel
-  2. if not found, grow the volumes with the right replication level
-  3. return a writable volume to the user
-
-  
-  for data node:
-    0. detect existing volumes DONE
-    1. onStartUp, and periodically, send existing volumes and maxVolumeCount  store.Join(), DONE
-    2. accept command to grow a volume( id + replication level)  DONE
-       /admin/assign_volume?volume=some_id&replicationType=01
-    3. accept setting volumeLocationList  DONE
-       /admin/set_volume_locations_list?volumeLocationsList=[{Vid:xxx,Locations:[loc1,loc2,loc3]}]
-    4. for each write, pass the write to the next location, (Step 2)
-       POST method should accept an index, like ttl, get decremented every hop
-  for master:
-    1. accept data node's report of existing volumes and maxVolumeCount ALREADY EXISTS /dir/join
-    2. periodically refresh for active data nodes, and adjust writable volumes
-    3. send command to grow a volume(id + replication level)  DONE
-    5. accept lookup for volume locations    ALREADY EXISTS /dir/lookup
-    6. read topology/datacenter/rack layout
-
-An algorithm to allocate volumes evenly, but may be inefficient if free volumes are plenty:
-input: replication=xyz
-algorithm:
-ret_dcs = []
-foreach dc that has y+z+1 volumes{
-  ret_racks = []
-  foreach rack with z+1 volumes{
-    ret = select z+1 servers with 1 volume
-  if ret.size()==z+1 {
-    ret_racks.append(ret)
-  }
-  }
-  randomly pick one rack from ret_racks
-  ret += select y racks with 1 volume each
-  if ret.size()==y+z+1{
-    ret_dcs.append(ret)
-  }
-}
-randomly pick one dc from ret_dcs
-ret += select x data centers with 1 volume each
-
-A simple replica placement algorithm, but may fail when free volume slots are not plenty:
-ret := []volumes
-dc = randomly pick 1 data center with y+z+1 volumes
-  rack = randomly pick 1 rack with z+1 volumes
-    ret = ret.append(randomly pick z+1 volumes)
-  ret = ret.append(randomly pick y racks with 1 volume)
-ret = ret.append(randomly pick x data centers with 1 volume)
-
-
-TODO:
-  1. replicate content to the other server if the replication type needs replicas

note/security.txt

@@ -1,51 +0,0 @@
-Design for SeaweedFS security
-
-Design Objectives
-	Security can mean many different things. The original vision is that: if you have one machine lying around
-	somewhere with some disk space, it should be able to join your file system to contribute some disk space and
-	network bandwidth, securely!
-
-	To achieve this purpose, the security should be able to:
-	1. Secure the inter-server communication. Only real cluster servers can join and communicate.
-	2. allow clients to securely write to volume servers
-
-Non Objective
-	Multi-tenant support. Avoid filers or clients cross-updating files.
-	User specific access control.
-
-Design Architect
-	master, and volume servers all talk securely via 2-way SSL for admin operations.
-	upon joining, master gives its secret key to volume servers.
-	filer or clients talk to master to get secret key, and use the key to generate JWT to write on volume server.
-	A side benefit:
-		a time limited read feature?
-	4. volume server needs to expose https ports
-
-HTTP Connections
-	clear http
-		filer~>master, need to get a JWT from master
-		filer~>volume
-	2-way https
-		master~ssl~>volume
-		volume~ssl~>master
-
-file uploading:
-	when volume server starts, it asks master for the secret key to decode JWT
-	when filer/clients wants to upload, master generate a JWT
-		filer~>volume(public port)
-		master~>volume(public port)
-
-Currently, volume server has 2 ip addresses: ip and publicUrl.
-	The ip is for admin purpose, and master talk to volume server this way.
-	The publicUrl is for clients to access the server, via http GET/POST/DELETE etc.
-		The write operations are secured by JWT.
-	clients talk to master also via https? possible. Decide on this later.
-
-Dev plan:
-	1. volume server separate admin from public GET/POST/DELETE handlers
-		The step 1 may be good enough for most use cases.
-
-	If 2-way ssl are still needed
-	2. volume server add ssl support
-	3. https connections to operate on volume servers
-

note/weedfs.txt

@@ -1,46 +0,0 @@
-How to submit a content
-1. Find physical volumes
-1.c Create a hash value
-1.d find a write logic volume id, and return [logic volume id, {physical volume ids}]
-2. submit to physical volumes
-2.c
-  generate the cookie
-  generate a unique id as key
-  choose the right altKey
-  send bytes to physical volumes
-2.s each
-  save bytes
-  store map[key uint64, altKey uint32]<offset, size>
-    for updated entry, set old entry's offset to zero
-3.c
-  wait for all physical volumes to finish
-  store the /<logic volume id>/<key>_<cookie>_<altKey>.<ext>
-
-How to retrieve a content
-1.c
-  send logic volume id
-1.d
-  find least busy volume's id
-2.c
-  send URI /<physical volume id>/<key>_<cookie>_<altKey>.<ext>
-
-
-How to submit a content
-1. send bytes to SeaweedFS, got <volume id, key uint64, cookie code>
-   store <key uint64, volume id uint32, cookie code uint32, ext>, and other information
-
-To read a content
-2. use logic volume id to lookup a <machine id>
-   render url as /<machine id>/<volume id>/<key>/<cookie>.ext
-
-The directory server
-0.init
-  load and collect <logic volume id, machine ids> mapping
-1.on submit content
-  find a free logic volume id, start sending content to 3 machines
-  if all of them finishes, return <logic volume id, key, cookie code>
-2.on read content
-  based on logic volume id, pick a machine with less load,
-    return <machine id>
-
-

other/java/client/pom.xml

@@ -0,0 +1,161 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.github.chrislusf</groupId>
+    <artifactId>seaweedfs-client</artifactId>
+    <version>1.0.9</version>
+
+    <parent>
+        <groupId>org.sonatype.oss</groupId>
+        <artifactId>oss-parent</artifactId>
+        <version>9</version>
+    </parent>
+
+    <properties>
+        <protobuf.version>3.5.1</protobuf.version>
+        <grpc.version>1.16.1</grpc.version>
+        <guava.version>26.0-jre</guava.version>
+    </properties>
+
+    <dependencies>
+        <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>${protobuf.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${guava.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-netty-shaded</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-protobuf</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-stub</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.25</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpmime</artifactId>
+            <version>4.5.6</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.12</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <distributionManagement>
+        <snapshotRepository>
+            <id>ossrh</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+        </snapshotRepository>
+    </distributionManagement>
+    <build>
+        <extensions>
+            <extension>
+                <groupId>kr.motd.maven</groupId>
+                <artifactId>os-maven-plugin</artifactId>
+                <version>1.5.0.Final</version>
+            </extension>
+        </extensions>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>7</source>
+                    <target>7</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.xolstice.maven.plugins</groupId>
+                <artifactId>protobuf-maven-plugin</artifactId>
+                <version>0.5.1</version>
+                <configuration>
+                    <protocArtifact>com.google.protobuf:protoc:${protobuf.version}-1:exe:${os.detected.classifier}</protocArtifact>
+                    <pluginId>grpc-java</pluginId>
+                    <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>compile-custom</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-gpg-plugin</artifactId>
+                <version>1.5</version>
+                <executions>
+                    <execution>
+                        <id>sign-artifacts</id>
+                        <phase>verify</phase>
+                        <goals>
+                            <goal>sign</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.sonatype.plugins</groupId>
+                <artifactId>nexus-staging-maven-plugin</artifactId>
+                <version>1.6.7</version>
+                <extensions>true</extensions>
+                <configuration>
+                    <serverId>ossrh</serverId>
+                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
+                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>2.2.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar-no-fork</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>2.9.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

other/java/client/src/main/java/seaweedfs/client/ByteBufferOutputStream.java

@@ -0,0 +1,21 @@
+package seaweedfs.client;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+public class ByteBufferOutputStream extends OutputStream {
+    private final ByteBuffer buf;
+
+    public ByteBufferOutputStream(ByteBuffer buf) {
+        this.buf = buf;
+    }
+
+    public void write(int b) throws IOException {
+        this.buf.put((byte)b);
+    }
+
+    public void write(byte[] b, int off, int len) throws IOException {
+        this.buf.put(b, off, len);
+    }
+}

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -0,0 +1,254 @@
+package seaweedfs.client;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class FilerClient {
+
+    private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
+
+    private FilerGrpcClient filerGrpcClient;
+
+    public FilerClient(String host, int grpcPort) {
+        filerGrpcClient = new FilerGrpcClient(host, grpcPort);
+    }
+
+    public FilerClient(FilerGrpcClient filerGrpcClient) {
+        this.filerGrpcClient = filerGrpcClient;
+    }
+
+    public boolean mkdirs(String path, int mode) {
+        String currentUser = System.getProperty("user.name");
+        return mkdirs(path, mode, 0, 0, currentUser, new String[]{});
+    }
+
+    public boolean mkdirs(String path, int mode, String userName, String[] groupNames) {
+        return mkdirs(path, mode, 0, 0, userName, groupNames);
+    }
+
+    public boolean mkdirs(String path, int mode, int uid, int gid, String userName, String[] groupNames) {
+
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();
+
+        if ("/".equals(path)) {
+            return true;
+        }
+
+        mkdirs(parent, mode, uid, gid, userName, groupNames);
+
+        FilerProto.Entry existingEntry = lookupEntry(parent, name);
+
+        if (existingEntry != null) {
+            return true;
+        }
+
+        return createEntry(
+                parent,
+                newDirectoryEntry(name, mode, uid, gid, userName, groupNames).build()
+        );
+
+    }
+
+    public boolean mv(String oldPath, String newPath) {
+
+        Path oldPathObject = Paths.get(oldPath);
+        String oldParent = oldPathObject.getParent().toString();
+        String oldName = oldPathObject.getFileName().toString();
+
+        Path newPathObject = Paths.get(newPath);
+        String newParent = newPathObject.getParent().toString();
+        String newName = newPathObject.getFileName().toString();
+
+        return atomicRenameEntry(oldParent, oldName, newParent, newName);
+
+    }
+
+    public boolean rm(String path, boolean isRecursive) {
+
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();
+
+        return deleteEntry(
+                parent,
+                name,
+                true,
+                isRecursive);
+    }
+
+    public boolean touch(String path, int mode) {
+        String currentUser = System.getProperty("user.name");
+        return touch(path, mode, 0, 0, currentUser, new String[]{});
+    }
+
+    public boolean touch(String path, int mode, int uid, int gid, String userName, String[] groupNames) {
+
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();
+
+        FilerProto.Entry entry = lookupEntry(parent, name);
+        if (entry == null) {
+            return createEntry(
+                    parent,
+                    newFileEntry(name, mode, uid, gid, userName, groupNames).build()
+            );
+        }
+        long now = System.currentTimeMillis() / 1000L;
+        FilerProto.FuseAttributes.Builder attr = entry.getAttributes().toBuilder()
+                .setMtime(now)
+                .setUid(uid)
+                .setGid(gid)
+                .setUserName(userName)
+                .clearGroupName()
+                .addAllGroupName(Arrays.asList(groupNames));
+        return updateEntry(parent, entry.toBuilder().setAttributes(attr).build());
+    }
+
+    public FilerProto.Entry.Builder newDirectoryEntry(String name, int mode,
+                                                      int uid, int gid, String userName, String[] groupNames) {
+
+        long now = System.currentTimeMillis() / 1000L;
+
+        return FilerProto.Entry.newBuilder()
+                .setName(name)
+                .setIsDirectory(true)
+                .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                        .setMtime(now)
+                        .setCrtime(now)
+                        .setUid(uid)
+                        .setGid(gid)
+                        .setFileMode(mode | 1 << 31)
+                        .setUserName(userName)
+                        .clearGroupName()
+                        .addAllGroupName(Arrays.asList(groupNames)));
+    }
+
+    public FilerProto.Entry.Builder newFileEntry(String name, int mode,
+                                                 int uid, int gid, String userName, String[] groupNames) {
+
+        long now = System.currentTimeMillis() / 1000L;
+
+        return FilerProto.Entry.newBuilder()
+                .setName(name)
+                .setIsDirectory(false)
+                .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                        .setMtime(now)
+                        .setCrtime(now)
+                        .setUid(uid)
+                        .setGid(gid)
+                        .setFileMode(mode)
+                        .setUserName(userName)
+                        .clearGroupName()
+                        .addAllGroupName(Arrays.asList(groupNames)));
+    }
+
+    public List<FilerProto.Entry> listEntries(String path) {
+        List<FilerProto.Entry> results = new ArrayList<FilerProto.Entry>();
+        String lastFileName = "";
+        for (int limit = Integer.MAX_VALUE; limit > 0; ) {
+            List<FilerProto.Entry> t = listEntries(path, "", lastFileName, 1024);
+            if (t == null) {
+                break;
+            }
+            int nSize = t.size();
+            if (nSize > 0) {
+                limit -= nSize;
+                lastFileName = t.get(nSize - 1).getName();
+            }
+            results.addAll(t);
+            if (t.size() < 1024) {
+                break;
+            }
+        }
+        return results;
+    }
+
+    public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) {
+        return filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
+                .setDirectory(path)
+                .setPrefix(entryPrefix)
+                .setStartFromFileName(lastEntryName)
+                .setLimit(limit)
+                .build()).getEntriesList();
+    }
+
+    public FilerProto.Entry lookupEntry(String directory, String entryName) {
+        try {
+            return filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
+                    FilerProto.LookupDirectoryEntryRequest.newBuilder()
+                            .setDirectory(directory)
+                            .setName(entryName)
+                            .build()).getEntry();
+        } catch (Exception e) {
+            LOG.warn("lookupEntry {}/{}: {}", directory, entryName, e);
+            return null;
+        }
+    }
+
+
+    public boolean createEntry(String parent, FilerProto.Entry entry) {
+        try {
+            filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
+                    .setDirectory(parent)
+                    .setEntry(entry)
+                    .build());
+        } catch (Exception e) {
+            LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
+            return false;
+        }
+        return true;
+    }
+
+    public boolean updateEntry(String parent, FilerProto.Entry entry) {
+        try {
+            filerGrpcClient.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder()
+                    .setDirectory(parent)
+                    .setEntry(entry)
+                    .build());
+        } catch (Exception e) {
+            LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
+            return false;
+        }
+        return true;
+    }
+
+    public boolean deleteEntry(String parent, String entryName, boolean isDeleteFileChunk, boolean isRecursive) {
+        try {
+            filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
+                    .setDirectory(parent)
+                    .setName(entryName)
+                    .setIsDeleteData(isDeleteFileChunk)
+                    .setIsRecursive(isRecursive)
+                    .build());
+        } catch (Exception e) {
+            LOG.warn("deleteEntry {}/{}: {}", parent, entryName, e);
+            return false;
+        }
+        return true;
+    }
+
+    public boolean atomicRenameEntry(String oldParent, String oldName, String newParent, String newName) {
+        try {
+            filerGrpcClient.getBlockingStub().atomicRenameEntry(FilerProto.AtomicRenameEntryRequest.newBuilder()
+                    .setOldDirectory(oldParent)
+                    .setOldName(oldName)
+                    .setNewDirectory(newParent)
+                    .setNewName(newName)
+                    .build());
+        } catch (Exception e) {
+            LOG.warn("atomicRenameEntry {}/{} => {}/{}: {}", oldParent, oldName, newParent, newName, e);
+            return false;
+        }
+        return true;
+    }
+
+}

other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java

@@ -0,0 +1,76 @@
+package seaweedfs.client;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
+import io.grpc.netty.shaded.io.grpc.netty.NegotiationType;
+import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
+
+import javax.net.ssl.SSLException;
+import java.io.File;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+public class FilerGrpcClient {
+
+    private static final Logger logger = Logger.getLogger(FilerGrpcClient.class.getName());
+
+    private final ManagedChannel channel;
+    private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub;
+    private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub;
+    private final SeaweedFilerGrpc.SeaweedFilerFutureStub futureStub;
+
+
+    public FilerGrpcClient(String host, int grpcPort) {
+        this(ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext());
+    }
+
+    public FilerGrpcClient(String host, int grpcPort,
+                           String caFilePath,
+                           String clientCertFilePath,
+                           String clientPrivateKeyFilePath) throws SSLException {
+
+        this(NettyChannelBuilder.forAddress(host, grpcPort)
+                .negotiationType(NegotiationType.TLS)
+                .sslContext(buildSslContext(caFilePath,clientCertFilePath,clientPrivateKeyFilePath)));
+    }
+
+    public FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {
+        channel = channelBuilder.build();
+        blockingStub = SeaweedFilerGrpc.newBlockingStub(channel);
+        asyncStub = SeaweedFilerGrpc.newStub(channel);
+        futureStub = SeaweedFilerGrpc.newFutureStub(channel);
+    }
+
+    public void shutdown() throws InterruptedException {
+        channel.shutdown().awaitTermination(5, TimeUnit.SECONDS);
+    }
+
+    public SeaweedFilerGrpc.SeaweedFilerBlockingStub getBlockingStub() {
+        return blockingStub;
+    }
+
+    public SeaweedFilerGrpc.SeaweedFilerStub getAsyncStub() {
+        return asyncStub;
+    }
+
+    public SeaweedFilerGrpc.SeaweedFilerFutureStub getFutureStub() {
+        return futureStub;
+    }
+
+    private static SslContext buildSslContext(String trustCertCollectionFilePath,
+                                              String clientCertChainFilePath,
+                                              String clientPrivateKeyFilePath) throws SSLException {
+        SslContextBuilder builder = GrpcSslContexts.forClient();
+        if (trustCertCollectionFilePath != null) {
+            builder.trustManager(new File(trustCertCollectionFilePath));
+        }
+        if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) {
+            builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath));
+        }
+        return builder.build();
+    }
+
+}

other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@@ -0,0 +1,260 @@
+package seaweedfs.client;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.HttpClientBuilder;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
+public class SeaweedRead {
+
+    // private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
+
+    // returns bytesRead
+    public static long read(FilerGrpcClient filerGrpcClient, List<VisibleInterval> visibleIntervals,
+                            final long position, final byte[] buffer, final int bufferOffset,
+                            final int bufferLength) {
+
+        List<ChunkView> chunkViews = viewFromVisibles(visibleIntervals, position, bufferLength);
+
+        FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder();
+        for (ChunkView chunkView : chunkViews) {
+            String vid = parseVolumeId(chunkView.fileId);
+            lookupRequest.addVolumeIds(vid);
+        }
+
+        FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient
+            .getBlockingStub().lookupVolume(lookupRequest.build());
+
+        Map<String, FilerProto.Locations> vid2Locations = lookupResponse.getLocationsMapMap();
+
+        //TODO parallel this
+        long readCount = 0;
+        int startOffset = bufferOffset;
+        for (ChunkView chunkView : chunkViews) {
+            FilerProto.Locations locations = vid2Locations.get(parseVolumeId(chunkView.fileId));
+            if (locations.getLocationsCount() == 0) {
+                // log here!
+                return 0;
+            }
+
+            int len = readChunkView(position, buffer, startOffset, chunkView, locations);
+
+            readCount += len;
+            startOffset += len;
+
+        }
+
+        return readCount;
+    }
+
+    private static int readChunkView(long position, byte[] buffer, int startOffset, ChunkView chunkView, FilerProto.Locations locations) {
+        HttpClient client = HttpClientBuilder.create().build();
+        HttpGet request = new HttpGet(
+            String.format("http://%s/%s", locations.getLocations(0).getUrl(), chunkView.fileId));
+
+        if (!chunkView.isFullChunk) {
+            request.setHeader(HttpHeaders.ACCEPT_ENCODING, "");
+            request.setHeader(HttpHeaders.RANGE,
+                String.format("bytes=%d-%d", chunkView.offset, chunkView.offset + chunkView.size));
+        }
+
+        try {
+            HttpResponse response = client.execute(request);
+            HttpEntity entity = response.getEntity();
+
+            int len = (int) (chunkView.logicOffset - position + chunkView.size);
+            OutputStream outputStream = new ByteBufferOutputStream(ByteBuffer.wrap(buffer, startOffset, len));
+            entity.writeTo(outputStream);
+            // LOG.debug("* read chunkView:{} startOffset:{} length:{}", chunkView, startOffset, len);
+
+            return len;
+
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return 0;
+    }
+
+    public static List<ChunkView> viewFromVisibles(List<VisibleInterval> visibleIntervals, long offset, long size) {
+        List<ChunkView> views = new ArrayList<>();
+
+        long stop = offset + size;
+        for (VisibleInterval chunk : visibleIntervals) {
+            if (chunk.start <= offset && offset < chunk.stop && offset < stop) {
+                boolean isFullChunk = chunk.isFullChunk && chunk.start == offset && chunk.stop <= stop;
+                views.add(new ChunkView(
+                    chunk.fileId,
+                    offset - chunk.start,
+                    Math.min(chunk.stop, stop) - offset,
+                    offset,
+                    isFullChunk
+                ));
+                offset = Math.min(chunk.stop, stop);
+            }
+        }
+        return views;
+    }
+
+    public static List<VisibleInterval> nonOverlappingVisibleIntervals(List<FilerProto.FileChunk> chunkList) {
+        FilerProto.FileChunk[] chunks = chunkList.toArray(new FilerProto.FileChunk[0]);
+        Arrays.sort(chunks, new Comparator<FilerProto.FileChunk>() {
+            @Override
+            public int compare(FilerProto.FileChunk a, FilerProto.FileChunk b) {
+                return (int) (a.getMtime() - b.getMtime());
+            }
+        });
+
+        List<VisibleInterval> visibles = new ArrayList<>();
+        for (FilerProto.FileChunk chunk : chunks) {
+            List<VisibleInterval> newVisibles = new ArrayList<>();
+            visibles = mergeIntoVisibles(visibles, newVisibles, chunk);
+        }
+
+        return visibles;
+    }
+
+    private static List<VisibleInterval> mergeIntoVisibles(List<VisibleInterval> visibles,
+                                                           List<VisibleInterval> newVisibles,
+                                                           FilerProto.FileChunk chunk) {
+        VisibleInterval newV = new VisibleInterval(
+            chunk.getOffset(),
+            chunk.getOffset() + chunk.getSize(),
+            chunk.getFileId(),
+            chunk.getMtime(),
+            true
+        );
+
+        // easy cases to speed up
+        if (visibles.size() == 0) {
+            visibles.add(newV);
+            return visibles;
+        }
+        if (visibles.get(visibles.size() - 1).stop <= chunk.getOffset()) {
+            visibles.add(newV);
+            return visibles;
+        }
+
+        for (VisibleInterval v : visibles) {
+            if (v.start < chunk.getOffset() && chunk.getOffset() < v.stop) {
+                newVisibles.add(new VisibleInterval(
+                    v.start,
+                    chunk.getOffset(),
+                    v.fileId,
+                    v.modifiedTime,
+                    false
+                ));
+            }
+            long chunkStop = chunk.getOffset() + chunk.getSize();
+            if (v.start < chunkStop && chunkStop < v.stop) {
+                newVisibles.add(new VisibleInterval(
+                    chunkStop,
+                    v.stop,
+                    v.fileId,
+                    v.modifiedTime,
+                    false
+                ));
+            }
+            if (chunkStop <= v.start || v.stop <= chunk.getOffset()) {
+                newVisibles.add(v);
+            }
+        }
+        newVisibles.add(newV);
+
+        // keep everything sorted
+        for (int i = newVisibles.size() - 1; i >= 0; i--) {
+            if (i > 0 && newV.start < newVisibles.get(i - 1).start) {
+                newVisibles.set(i, newVisibles.get(i - 1));
+            } else {
+                newVisibles.set(i, newV);
+                break;
+            }
+        }
+
+        return newVisibles;
+    }
+
+    public static String parseVolumeId(String fileId) {
+        int commaIndex = fileId.lastIndexOf(',');
+        if (commaIndex > 0) {
+            return fileId.substring(0, commaIndex);
+        }
+        return fileId;
+    }
+
+    public static long totalSize(List<FilerProto.FileChunk> chunksList) {
+        long size = 0;
+        for (FilerProto.FileChunk chunk : chunksList) {
+            long t = chunk.getOffset() + chunk.getSize();
+            if (size < t) {
+                size = t;
+            }
+        }
+        return size;
+    }
+
+    public static class VisibleInterval {
+        public final long start;
+        public final long stop;
+        public final long modifiedTime;
+        public final String fileId;
+        public final boolean isFullChunk;
+
+        public VisibleInterval(long start, long stop, String fileId, long modifiedTime, boolean isFullChunk) {
+            this.start = start;
+            this.stop = stop;
+            this.modifiedTime = modifiedTime;
+            this.fileId = fileId;
+            this.isFullChunk = isFullChunk;
+        }
+
+        @Override
+        public String toString() {
+            return "VisibleInterval{" +
+                "start=" + start +
+                ", stop=" + stop +
+                ", modifiedTime=" + modifiedTime +
+                ", fileId='" + fileId + '\'' +
+                ", isFullChunk=" + isFullChunk +
+                '}';
+        }
+    }
+
+    public static class ChunkView {
+        public final String fileId;
+        public final long offset;
+        public final long size;
+        public final long logicOffset;
+        public final boolean isFullChunk;
+
+        public ChunkView(String fileId, long offset, long size, long logicOffset, boolean isFullChunk) {
+            this.fileId = fileId;
+            this.offset = offset;
+            this.size = size;
+            this.logicOffset = logicOffset;
+            this.isFullChunk = isFullChunk;
+        }
+
+        @Override
+        public String toString() {
+            return "ChunkView{" +
+                "fileId='" + fileId + '\'' +
+                ", offset=" + offset +
+                ", size=" + size +
+                ", logicOffset=" + logicOffset +
+                ", isFullChunk=" + isFullChunk +
+                '}';
+        }
+    }
+
+}

other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java

@@ -0,0 +1,91 @@
+package seaweedfs.client;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.mime.HttpMultipartMode;
+import org.apache.http.entity.mime.MultipartEntityBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class SeaweedWrite {
+
+    public static void writeData(FilerProto.Entry.Builder entry,
+                                 final String replication,
+                                 final FilerGrpcClient filerGrpcClient,
+                                 final long offset,
+                                 final byte[] bytes,
+                                 final long bytesOffset, final long bytesLength) throws IOException {
+        FilerProto.AssignVolumeResponse response = filerGrpcClient.getBlockingStub().assignVolume(
+                FilerProto.AssignVolumeRequest.newBuilder()
+                        .setCollection("")
+                        .setReplication(replication)
+                        .setDataCenter("")
+                        .setReplication("")
+                        .setTtlSec(0)
+                        .build());
+        String fileId = response.getFileId();
+        String url = response.getUrl();
+        String auth = response.getAuth();
+        String targetUrl = String.format("http://%s/%s", url, fileId);
+
+        String etag = multipartUpload(targetUrl, auth, bytes, bytesOffset, bytesLength);
+
+        entry.addChunks(FilerProto.FileChunk.newBuilder()
+                .setFileId(fileId)
+                .setOffset(offset)
+                .setSize(bytesLength)
+                .setMtime(System.currentTimeMillis() / 10000L)
+                .setETag(etag)
+        );
+
+    }
+
+    public static void writeMeta(final FilerGrpcClient filerGrpcClient,
+                                 final String parentDirectory, final FilerProto.Entry.Builder entry) {
+        filerGrpcClient.getBlockingStub().createEntry(
+                FilerProto.CreateEntryRequest.newBuilder()
+                        .setDirectory(parentDirectory)
+                        .setEntry(entry)
+                        .build()
+        );
+    }
+
+    private static String multipartUpload(String targetUrl,
+                                          String auth,
+                                          final byte[] bytes,
+                                          final long bytesOffset, final long bytesLength) throws IOException {
+
+        CloseableHttpClient client = HttpClientBuilder.create().setUserAgent("hdfs-client").build();
+
+        InputStream inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
+
+        HttpPost post = new HttpPost(targetUrl);
+        if (auth != null && auth.length() != 0) {
+            post.addHeader("Authorization", "BEARER " + auth);
+        }
+
+        post.setEntity(MultipartEntityBuilder.create()
+                .setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
+                .addBinaryBody("upload", inputStream)
+                .build());
+
+        try {
+            HttpResponse response = client.execute(post);
+
+            String etag = response.getLastHeader("ETag").getValue();
+
+            if (etag != null && etag.startsWith("\"") && etag.endsWith("\"")) {
+                etag = etag.substring(1, etag.length() - 1);
+            }
+
+            return etag;
+        } finally {
+            client.close();
+        }
+
+    }
+}

other/java/client/src/main/proto/filer.proto

@@ -0,0 +1,199 @@
+syntax = "proto3";
+
+package filer_pb;
+
+option java_package = "seaweedfs.client";
+option java_outer_classname = "FilerProto";
+
+//////////////////////////////////////////////////
+
+service SeaweedFiler {
+
+    rpc LookupDirectoryEntry (LookupDirectoryEntryRequest) returns (LookupDirectoryEntryResponse) {
+    }
+
+    rpc ListEntries (ListEntriesRequest) returns (ListEntriesResponse) {
+    }
+
+    rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) {
+    }
+
+    rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) {
+    }
+
+    rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
+    }
+
+    rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
+    }
+
+    rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
+    }
+
+    rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
+    }
+
+    rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
+    }
+
+    rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
+    }
+
+}
+
+//////////////////////////////////////////////////
+
+message LookupDirectoryEntryRequest {
+    string directory = 1;
+    string name = 2;
+}
+
+message LookupDirectoryEntryResponse {
+    Entry entry = 1;
+}
+
+message ListEntriesRequest {
+    string directory = 1;
+    string prefix = 2;
+    string startFromFileName = 3;
+    bool inclusiveStartFrom = 4;
+    uint32 limit = 5;
+}
+
+message ListEntriesResponse {
+    repeated Entry entries = 1;
+}
+
+message Entry {
+    string name = 1;
+    bool is_directory = 2;
+    repeated FileChunk chunks = 3;
+    FuseAttributes attributes = 4;
+    map<string, bytes> extended = 5;
+}
+
+message FullEntry {
+    string dir = 1;
+    Entry entry = 2;
+}
+
+message EventNotification {
+    Entry old_entry = 1;
+    Entry new_entry = 2;
+    bool delete_chunks = 3;
+    string new_parent_path = 4;
+}
+
+message FileChunk {
+    string file_id = 1;
+    int64 offset = 2;
+    uint64 size = 3;
+    int64 mtime = 4;
+    string e_tag = 5;
+    string source_file_id = 6;
+}
+
+message FuseAttributes {
+    uint64 file_size = 1;
+    int64 mtime = 2; // unix time in seconds
+    uint32 file_mode = 3;
+    uint32 uid = 4;
+    uint32 gid = 5;
+    int64 crtime = 6; // unix time in seconds
+    string mime = 7;
+    string replication = 8;
+    string collection = 9;
+    int32 ttl_sec = 10;
+    string user_name = 11; // for hdfs
+    repeated string group_name = 12; // for hdfs
+    string symlink_target = 13;
+}
+
+message CreateEntryRequest {
+    string directory = 1;
+    Entry entry = 2;
+}
+
+message CreateEntryResponse {
+}
+
+message UpdateEntryRequest {
+    string directory = 1;
+    Entry entry = 2;
+}
+message UpdateEntryResponse {
+}
+
+message DeleteEntryRequest {
+    string directory = 1;
+    string name = 2;
+    // bool is_directory = 3;
+    bool is_delete_data = 4;
+    bool is_recursive = 5;
+}
+
+message DeleteEntryResponse {
+}
+
+message AtomicRenameEntryRequest {
+    string old_directory = 1;
+    string old_name = 2;
+    string new_directory = 3;
+    string new_name = 4;
+}
+
+message AtomicRenameEntryResponse {
+}
+
+message AssignVolumeRequest {
+    int32 count = 1;
+    string collection = 2;
+    string replication = 3;
+    int32 ttl_sec = 4;
+    string data_center = 5;
+}
+
+message AssignVolumeResponse {
+    string file_id = 1;
+    string url = 2;
+    string public_url = 3;
+    int32 count = 4;
+    string auth = 5;
+}
+
+message LookupVolumeRequest {
+    repeated string volume_ids = 1;
+}
+
+message Locations {
+    repeated Location locations = 1;
+}
+
+message Location {
+    string url = 1;
+    string public_url = 2;
+}
+message LookupVolumeResponse {
+    map<string, Locations> locations_map = 1;
+}
+
+message DeleteCollectionRequest {
+    string collection = 1;
+}
+
+message DeleteCollectionResponse {
+}
+
+message StatisticsRequest {
+    string replication = 1;
+    string collection = 2;
+    string ttl = 3;
+}
+message StatisticsResponse {
+    string replication = 1;
+    string collection = 2;
+    string ttl = 3;
+    uint64 total_size = 4;
+    uint64 used_size = 5;
+    uint64 file_count = 6;
+}

other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java

@@ -0,0 +1,63 @@
+package seaweedfs.client;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class SeaweedReadTest {
+
+    @Test
+    public void testNonOverlappingVisibleIntervals() {
+        List<FilerProto.FileChunk> chunks = new ArrayList<>();
+        chunks.add(FilerProto.FileChunk.newBuilder()
+            .setFileId("aaa")
+            .setOffset(0)
+            .setSize(100)
+            .setMtime(1000)
+            .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+            .setFileId("bbb")
+            .setOffset(100)
+            .setSize(133)
+            .setMtime(2000)
+            .build());
+
+        List<SeaweedRead.VisibleInterval> visibleIntervals = SeaweedRead.nonOverlappingVisibleIntervals(chunks);
+        for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) {
+            System.out.println("visible:" + visibleInterval);
+        }
+
+        Assert.assertEquals(visibleIntervals.size(), 2);
+
+        SeaweedRead.VisibleInterval visibleInterval = visibleIntervals.get(0);
+        Assert.assertEquals(visibleInterval.start, 0);
+        Assert.assertEquals(visibleInterval.stop, 100);
+        Assert.assertEquals(visibleInterval.modifiedTime, 1000);
+        Assert.assertEquals(visibleInterval.fileId, "aaa");
+
+        visibleInterval = visibleIntervals.get(1);
+        Assert.assertEquals(visibleInterval.start, 100);
+        Assert.assertEquals(visibleInterval.stop, 233);
+        Assert.assertEquals(visibleInterval.modifiedTime, 2000);
+        Assert.assertEquals(visibleInterval.fileId, "bbb");
+
+        List<SeaweedRead.ChunkView> chunkViews = SeaweedRead.viewFromVisibles(visibleIntervals, 0, 233);
+
+        SeaweedRead.ChunkView chunkView = chunkViews.get(0);
+        Assert.assertEquals(chunkView.offset, 0);
+        Assert.assertEquals(chunkView.size, 100);
+        Assert.assertEquals(chunkView.logicOffset, 0);
+        Assert.assertEquals(chunkView.fileId, "aaa");
+
+        chunkView = chunkViews.get(1);
+        Assert.assertEquals(chunkView.offset, 0);
+        Assert.assertEquals(chunkView.size, 133);
+        Assert.assertEquals(chunkView.logicOffset, 100);
+        Assert.assertEquals(chunkView.fileId, "bbb");
+
+
+    }
+
+}

other/java/hdfs/pom.xml

@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <properties>
+        <seaweedfs.client.version>1.0.9</seaweedfs.client.version>
+        <hadoop.version>3.1.1</hadoop.version>
+    </properties>
+
+    <groupId>com.github.chrislusf</groupId>
+    <artifactId>seaweedfs-hadoop-client</artifactId>
+    <version>${seaweedfs.client.version}</version>
+
+    <parent>
+        <groupId>org.sonatype.oss</groupId>
+        <artifactId>oss-parent</artifactId>
+        <version>9</version>
+    </parent>
+
+    <distributionManagement>
+        <snapshotRepository>
+            <id>ossrh</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+        </snapshotRepository>
+    </distributionManagement>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>7</source>
+                    <target>7</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.1</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                        <exclude>org/slf4j/**</exclude>
+                                        <exclude>META-INF/maven/org.slf4j/**</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                            </transformers>
+                            <relocations>
+                                <relocation>
+                                    <pattern>com.google</pattern>
+                                    <shadedPattern>shaded.com.google</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>io.grpc.internal</pattern>
+                                    <shadedPattern>shaded.io.grpc.internal</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>org.apache.commons</pattern>
+                                    <shadedPattern>shaded.org.apache.commons</shadedPattern>
+                                    <excludes>
+                                        <exclude>org.apache.hadoop</exclude>
+                                        <exclude>org.apache.log4j</exclude>
+                                    </excludes>
+                                </relocation>
+                            </relocations>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-gpg-plugin</artifactId>
+                <version>1.5</version>
+                <executions>
+                    <execution>
+                        <id>sign-artifacts</id>
+                        <phase>verify</phase>
+                        <goals>
+                            <goal>sign</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.sonatype.plugins</groupId>
+                <artifactId>nexus-staging-maven-plugin</artifactId>
+                <version>1.6.7</version>
+                <extensions>true</extensions>
+                <configuration>
+                    <serverId>ossrh</serverId>
+                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
+                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>2.2.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar-no-fork</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>2.9.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.github.chrislusf</groupId>
+            <artifactId>seaweedfs-client</artifactId>
+            <version>${seaweedfs.client.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+    </dependencies>
+
+</project>

other/java/hdfs/src/main/java/seaweed/hdfs/ReadBuffer.java

@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package seaweed.hdfs;
+
+import java.util.concurrent.CountDownLatch;
+
+class ReadBuffer {
+
+  private SeaweedInputStream stream;
+  private long offset;                   // offset within the file for the buffer
+  private int length;                    // actual length, set after the buffer is filles
+  private int requestedLength;           // requested length of the read
+  private byte[] buffer;                 // the buffer itself
+  private int bufferindex = -1;          // index in the buffers array in Buffer manager
+  private ReadBufferStatus status;             // status of the buffer
+  private CountDownLatch latch = null;   // signaled when the buffer is done reading, so any client
+  // waiting on this buffer gets unblocked
+
+  // fields to help with eviction logic
+  private long timeStamp = 0;  // tick at which buffer became available to read
+  private boolean isFirstByteConsumed = false;
+  private boolean isLastByteConsumed = false;
+  private boolean isAnyByteConsumed = false;
+
+  public SeaweedInputStream getStream() {
+    return stream;
+  }
+
+  public void setStream(SeaweedInputStream stream) {
+    this.stream = stream;
+  }
+
+  public long getOffset() {
+    return offset;
+  }
+
+  public void setOffset(long offset) {
+    this.offset = offset;
+  }
+
+  public int getLength() {
+    return length;
+  }
+
+  public void setLength(int length) {
+    this.length = length;
+  }
+
+  public int getRequestedLength() {
+    return requestedLength;
+  }
+
+  public void setRequestedLength(int requestedLength) {
+    this.requestedLength = requestedLength;
+  }
+
+  public byte[] getBuffer() {
+    return buffer;
+  }
+
+  public void setBuffer(byte[] buffer) {
+    this.buffer = buffer;
+  }
+
+  public int getBufferindex() {
+    return bufferindex;
+  }
+
+  public void setBufferindex(int bufferindex) {
+    this.bufferindex = bufferindex;
+  }
+
+  public ReadBufferStatus getStatus() {
+    return status;
+  }
+
+  public void setStatus(ReadBufferStatus status) {
+    this.status = status;
+  }
+
+  public CountDownLatch getLatch() {
+    return latch;
+  }
+
+  public void setLatch(CountDownLatch latch) {
+    this.latch = latch;
+  }
+
+  public long getTimeStamp() {
+    return timeStamp;
+  }
+
+  public void setTimeStamp(long timeStamp) {
+    this.timeStamp = timeStamp;
+  }
+
+  public boolean isFirstByteConsumed() {
+    return isFirstByteConsumed;
+  }
+
+  public void setFirstByteConsumed(boolean isFirstByteConsumed) {
+    this.isFirstByteConsumed = isFirstByteConsumed;
+  }
+
+  public boolean isLastByteConsumed() {
+    return isLastByteConsumed;
+  }
+
+  public void setLastByteConsumed(boolean isLastByteConsumed) {
+    this.isLastByteConsumed = isLastByteConsumed;
+  }
+
+  public boolean isAnyByteConsumed() {
+    return isAnyByteConsumed;
+  }
+
+  public void setAnyByteConsumed(boolean isAnyByteConsumed) {
+    this.isAnyByteConsumed = isAnyByteConsumed;
+  }
+
+}

other/java/hdfs/src/main/java/seaweed/hdfs/ReadBufferManager.java

@@ -0,0 +1,394 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package seaweed.hdfs;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.Stack;
+import java.util.concurrent.CountDownLatch;
+
+/**
+ * The Read Buffer Manager for Rest AbfsClient.
+ */
+final class ReadBufferManager {
+  private static final Logger LOGGER = LoggerFactory.getLogger(ReadBufferManager.class);
+
+  private static final int NUM_BUFFERS = 16;
+  private static final int BLOCK_SIZE = 4 * 1024 * 1024;
+  private static final int NUM_THREADS = 8;
+  private static final int THRESHOLD_AGE_MILLISECONDS = 3000; // have to see if 3 seconds is a good threshold
+
+  private Thread[] threads = new Thread[NUM_THREADS];
+  private byte[][] buffers;    // array of byte[] buffers, to hold the data that is read
+  private Stack<Integer> freeList = new Stack<>();   // indices in buffers[] array that are available
+
+  private Queue<ReadBuffer> readAheadQueue = new LinkedList<>(); // queue of requests that are not picked up by any worker thread yet
+  private LinkedList<ReadBuffer> inProgressList = new LinkedList<>(); // requests being processed by worker threads
+  private LinkedList<ReadBuffer> completedReadList = new LinkedList<>(); // buffers available for reading
+  private static final ReadBufferManager BUFFER_MANAGER; // singleton, initialized in static initialization block
+
+  static {
+    BUFFER_MANAGER = new ReadBufferManager();
+    BUFFER_MANAGER.init();
+  }
+
+  static ReadBufferManager getBufferManager() {
+    return BUFFER_MANAGER;
+  }
+
+  private void init() {
+    buffers = new byte[NUM_BUFFERS][];
+    for (int i = 0; i < NUM_BUFFERS; i++) {
+      buffers[i] = new byte[BLOCK_SIZE];  // same buffers are reused. The byte array never goes back to GC
+      freeList.add(i);
+    }
+    for (int i = 0; i < NUM_THREADS; i++) {
+      Thread t = new Thread(new ReadBufferWorker(i));
+      t.setDaemon(true);
+      threads[i] = t;
+      t.setName("SeaweedFS-prefetch-" + i);
+      t.start();
+    }
+    ReadBufferWorker.UNLEASH_WORKERS.countDown();
+  }
+
+  // hide instance constructor
+  private ReadBufferManager() {
+  }
+
+
+  /*
+   *
+   *  SeaweedInputStream-facing methods
+   *
+   */
+
+
+  /**
+   * {@link SeaweedInputStream} calls this method to queue read-aheads.
+   *
+   * @param stream          The {@link SeaweedInputStream} for which to do the read-ahead
+   * @param requestedOffset The offset in the file which shoukd be read
+   * @param requestedLength The length to read
+   */
+  void queueReadAhead(final SeaweedInputStream stream, final long requestedOffset, final int requestedLength) {
+    if (LOGGER.isTraceEnabled()) {
+      LOGGER.trace("Start Queueing readAhead for {} offset {} length {}",
+          stream.getPath(), requestedOffset, requestedLength);
+    }
+    ReadBuffer buffer;
+    synchronized (this) {
+      if (isAlreadyQueued(stream, requestedOffset)) {
+        return; // already queued, do not queue again
+      }
+      if (freeList.isEmpty() && !tryEvict()) {
+        return; // no buffers available, cannot queue anything
+      }
+
+      buffer = new ReadBuffer();
+      buffer.setStream(stream);
+      buffer.setOffset(requestedOffset);
+      buffer.setLength(0);
+      buffer.setRequestedLength(requestedLength);
+      buffer.setStatus(ReadBufferStatus.NOT_AVAILABLE);
+      buffer.setLatch(new CountDownLatch(1));
+
+      Integer bufferIndex = freeList.pop();  // will return a value, since we have checked size > 0 already
+
+      buffer.setBuffer(buffers[bufferIndex]);
+      buffer.setBufferindex(bufferIndex);
+      readAheadQueue.add(buffer);
+      notifyAll();
+    }
+    if (LOGGER.isTraceEnabled()) {
+      LOGGER.trace("Done q-ing readAhead for file {} offset {} buffer idx {}",
+          stream.getPath(), requestedOffset, buffer.getBufferindex());
+    }
+  }
+
+
+  /**
+   * {@link SeaweedInputStream} calls this method read any bytes already available in a buffer (thereby saving a
+   * remote read). This returns the bytes if the data already exists in buffer. If there is a buffer that is reading
+   * the requested offset, then this method blocks until that read completes. If the data is queued in a read-ahead
+   * but not picked up by a worker thread yet, then it cancels that read-ahead and reports cache miss. This is because
+   * depending on worker thread availability, the read-ahead may take a while - the calling thread can do it's own
+   * read to get the data faster (copmared to the read waiting in queue for an indeterminate amount of time).
+   *
+   * @param stream   the file to read bytes for
+   * @param position the offset in the file to do a read for
+   * @param length   the length to read
+   * @param buffer   the buffer to read data into. Note that the buffer will be written into from offset 0.
+   * @return the number of bytes read
+   */
+  int getBlock(final SeaweedInputStream stream, final long position, final int length, final byte[] buffer) {
+    // not synchronized, so have to be careful with locking
+    if (LOGGER.isTraceEnabled()) {
+      LOGGER.trace("getBlock for file {}  position {}  thread {}",
+          stream.getPath(), position, Thread.currentThread().getName());
+    }
+
+    waitForProcess(stream, position);
+
+    int bytesRead = 0;
+    synchronized (this) {
+      bytesRead = getBlockFromCompletedQueue(stream, position, length, buffer);
+    }
+    if (bytesRead > 0) {
+      if (LOGGER.isTraceEnabled()) {
+        LOGGER.trace("Done read from Cache for {} position {} length {}",
+            stream.getPath(), position, bytesRead);
+      }
+      return bytesRead;
+    }
+
+    // otherwise, just say we got nothing - calling thread can do its own read
+    return 0;
+  }
+
+  /*
+   *
+   *  Internal methods
+   *
+   */
+
+  private void waitForProcess(final SeaweedInputStream stream, final long position) {
+    ReadBuffer readBuf;
+    synchronized (this) {
+      clearFromReadAheadQueue(stream, position);
+      readBuf = getFromList(inProgressList, stream, position);
+    }
+    if (readBuf != null) {         // if in in-progress queue, then block for it
+      try {
+        if (LOGGER.isTraceEnabled()) {
+          LOGGER.trace("got a relevant read buffer for file {} offset {} buffer idx {}",
+              stream.getPath(), readBuf.getOffset(), readBuf.getBufferindex());
+        }
+        readBuf.getLatch().await();  // blocking wait on the caller stream's thread
+        // Note on correctness: readBuf gets out of inProgressList only in 1 place: after worker thread
+        // is done processing it (in doneReading). There, the latch is set after removing the buffer from
+        // inProgressList. So this latch is safe to be outside the synchronized block.
+        // Putting it in synchronized would result in a deadlock, since this thread would be holding the lock
+        // while waiting, so no one will be able to  change any state. If this becomes more complex in the future,
+        // then the latch cane be removed and replaced with wait/notify whenever inProgressList is touched.
+      } catch (InterruptedException ex) {
+        Thread.currentThread().interrupt();
+      }
+      if (LOGGER.isTraceEnabled()) {
+        LOGGER.trace("latch done for file {} buffer idx {} length {}",
+            stream.getPath(), readBuf.getBufferindex(), readBuf.getLength());
+      }
+    }
+  }
+
+  /**
+   * If any buffer in the completedlist can be reclaimed then reclaim it and return the buffer to free list.
+   * The objective is to find just one buffer - there is no advantage to evicting more than one.
+   *
+   * @return whether the eviction succeeeded - i.e., were we able to free up one buffer
+   */
+  private synchronized boolean tryEvict() {
+    ReadBuffer nodeToEvict = null;
+    if (completedReadList.size() <= 0) {
+      return false;  // there are no evict-able buffers
+    }
+
+    // first, try buffers where all bytes have been consumed (approximated as first and last bytes consumed)
+    for (ReadBuffer buf : completedReadList) {
+      if (buf.isFirstByteConsumed() && buf.isLastByteConsumed()) {
+        nodeToEvict = buf;
+        break;
+      }
+    }
+    if (nodeToEvict != null) {
+      return evict(nodeToEvict);
+    }
+
+    // next, try buffers where any bytes have been consumed (may be a bad idea? have to experiment and see)
+    for (ReadBuffer buf : completedReadList) {
+      if (buf.isAnyByteConsumed()) {
+        nodeToEvict = buf;
+        break;
+      }
+    }
+
+    if (nodeToEvict != null) {
+      return evict(nodeToEvict);
+    }
+
+    // next, try any old nodes that have not been consumed
+    long earliestBirthday = Long.MAX_VALUE;
+    for (ReadBuffer buf : completedReadList) {
+      if (buf.getTimeStamp() < earliestBirthday) {
+        nodeToEvict = buf;
+        earliestBirthday = buf.getTimeStamp();
+      }
+    }
+    if ((currentTimeMillis() - earliestBirthday > THRESHOLD_AGE_MILLISECONDS) && (nodeToEvict != null)) {
+      return evict(nodeToEvict);
+    }
+
+    // nothing can be evicted
+    return false;
+  }
+
+  private boolean evict(final ReadBuffer buf) {
+    freeList.push(buf.getBufferindex());
+    completedReadList.remove(buf);
+    if (LOGGER.isTraceEnabled()) {
+      LOGGER.trace("Evicting buffer idx {}; was used for file {} offset {} length {}",
+          buf.getBufferindex(), buf.getStream().getPath(), buf.getOffset(), buf.getLength());
+    }
+    return true;
+  }
+
+  private boolean isAlreadyQueued(final SeaweedInputStream stream, final long requestedOffset) {
+    // returns true if any part of the buffer is already queued
+    return (isInList(readAheadQueue, stream, requestedOffset)
+        || isInList(inProgressList, stream, requestedOffset)
+        || isInList(completedReadList, stream, requestedOffset));
+  }
+
+  private boolean isInList(final Collection<ReadBuffer> list, final SeaweedInputStream stream, final long requestedOffset) {
+    return (getFromList(list, stream, requestedOffset) != null);
+  }
+
+  private ReadBuffer getFromList(final Collection<ReadBuffer> list, final SeaweedInputStream stream, final long requestedOffset) {
+    for (ReadBuffer buffer : list) {
+      if (buffer.getStream() == stream) {
+        if (buffer.getStatus() == ReadBufferStatus.AVAILABLE
+            && requestedOffset >= buffer.getOffset()
+            && requestedOffset < buffer.getOffset() + buffer.getLength()) {
+          return buffer;
+        } else if (requestedOffset >= buffer.getOffset()
+            && requestedOffset < buffer.getOffset() + buffer.getRequestedLength()) {
+          return buffer;
+        }
+      }
+    }
+    return null;
+  }
+
+  private void clearFromReadAheadQueue(final SeaweedInputStream stream, final long requestedOffset) {
+    ReadBuffer buffer = getFromList(readAheadQueue, stream, requestedOffset);
+    if (buffer != null) {
+      readAheadQueue.remove(buffer);
+      notifyAll();   // lock is held in calling method
+      freeList.push(buffer.getBufferindex());
+    }
+  }
+
+  private int getBlockFromCompletedQueue(final SeaweedInputStream stream, final long position, final int length,
+                                         final byte[] buffer) {
+    ReadBuffer buf = getFromList(completedReadList, stream, position);
+    if (buf == null || position >= buf.getOffset() + buf.getLength()) {
+      return 0;
+    }
+    int cursor = (int) (position - buf.getOffset());
+    int availableLengthInBuffer = buf.getLength() - cursor;
+    int lengthToCopy = Math.min(length, availableLengthInBuffer);
+    System.arraycopy(buf.getBuffer(), cursor, buffer, 0, lengthToCopy);
+    if (cursor == 0) {
+      buf.setFirstByteConsumed(true);
+    }
+    if (cursor + lengthToCopy == buf.getLength()) {
+      buf.setLastByteConsumed(true);
+    }
+    buf.setAnyByteConsumed(true);
+    return lengthToCopy;
+  }
+
+  /*
+   *
+   *  ReadBufferWorker-thread-facing methods
+   *
+   */
+
+  /**
+   * ReadBufferWorker thread calls this to get the next buffer that it should work on.
+   *
+   * @return {@link ReadBuffer}
+   * @throws InterruptedException if thread is interrupted
+   */
+  ReadBuffer getNextBlockToRead() throws InterruptedException {
+    ReadBuffer buffer = null;
+    synchronized (this) {
+      //buffer = readAheadQueue.take();  // blocking method
+      while (readAheadQueue.size() == 0) {
+        wait();
+      }
+      buffer = readAheadQueue.remove();
+      notifyAll();
+      if (buffer == null) {
+        return null;            // should never happen
+      }
+      buffer.setStatus(ReadBufferStatus.READING_IN_PROGRESS);
+      inProgressList.add(buffer);
+    }
+    if (LOGGER.isTraceEnabled()) {
+      LOGGER.trace("ReadBufferWorker picked file {} for offset {}",
+          buffer.getStream().getPath(), buffer.getOffset());
+    }
+    return buffer;
+  }
+
+  /**
+   * ReadBufferWorker thread calls this method to post completion.
+   *
+   * @param buffer            the buffer whose read was completed
+   * @param result            the {@link ReadBufferStatus} after the read operation in the worker thread
+   * @param bytesActuallyRead the number of bytes that the worker thread was actually able to read
+   */
+  void doneReading(final ReadBuffer buffer, final ReadBufferStatus result, final int bytesActuallyRead) {
+    if (LOGGER.isTraceEnabled()) {
+      LOGGER.trace("ReadBufferWorker completed file {} for offset {} bytes {}",
+          buffer.getStream().getPath(),  buffer.getOffset(), bytesActuallyRead);
+    }
+    synchronized (this) {
+      inProgressList.remove(buffer);
+      if (result == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) {
+        buffer.setStatus(ReadBufferStatus.AVAILABLE);
+        buffer.setTimeStamp(currentTimeMillis());
+        buffer.setLength(bytesActuallyRead);
+        completedReadList.add(buffer);
+      } else {
+        freeList.push(buffer.getBufferindex());
+        // buffer should go out of scope after the end of the calling method in ReadBufferWorker, and eligible for GC
+      }
+    }
+    //outside the synchronized, since anyone receiving a wake-up from the latch must see safe-published results
+    buffer.getLatch().countDown(); // wake up waiting threads (if any)
+  }
+
+  /**
+   * Similar to System.currentTimeMillis, except implemented with System.nanoTime().
+   * System.currentTimeMillis can go backwards when system clock is changed (e.g., with NTP time synchronization),
+   * making it unsuitable for measuring time intervals. nanotime is strictly monotonically increasing per CPU core.
+   * Note: it is not monotonic across Sockets, and even within a CPU, its only the
+   * more recent parts which share a clock across all cores.
+   *
+   * @return current time in milliseconds
+   */
+  private long currentTimeMillis() {
+    return System.nanoTime() / 1000 / 1000;
+  }
+}

other/java/hdfs/src/main/java/seaweed/hdfs/ReadBufferStatus.java

@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package seaweed.hdfs;
+
+/**
+ * The ReadBufferStatus for Rest AbfsClient
+ */
+public enum ReadBufferStatus {
+  NOT_AVAILABLE,  // buffers sitting in readaheadqueue have this stats
+  READING_IN_PROGRESS,  // reading is in progress on this buffer. Buffer should be in inProgressList
+  AVAILABLE,  // data is available in buffer. It should be in completedList
+  READ_FAILED  // read completed, but failed.
+}

other/java/hdfs/src/main/java/seaweed/hdfs/ReadBufferWorker.java

@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package seaweed.hdfs;
+
+import java.util.concurrent.CountDownLatch;
+
+class ReadBufferWorker implements Runnable {
+
+  protected static final CountDownLatch UNLEASH_WORKERS = new CountDownLatch(1);
+  private int id;
+
+  ReadBufferWorker(final int id) {
+    this.id = id;
+  }
+
+  /**
+   * return the ID of ReadBufferWorker.
+   */
+  public int getId() {
+    return this.id;
+  }
+
+  /**
+   * Waits until a buffer becomes available in ReadAheadQueue.
+   * Once a buffer becomes available, reads the file specified in it and then posts results back to buffer manager.
+   * Rinse and repeat. Forever.
+   */
+  public void run() {
+    try {
+      UNLEASH_WORKERS.await();
+    } catch (InterruptedException ex) {
+      Thread.currentThread().interrupt();
+    }
+    ReadBufferManager bufferManager = ReadBufferManager.getBufferManager();
+    ReadBuffer buffer;
+    while (true) {
+      try {
+        buffer = bufferManager.getNextBlockToRead();   // blocks, until a buffer is available for this thread
+      } catch (InterruptedException ex) {
+        Thread.currentThread().interrupt();
+        return;
+      }
+      if (buffer != null) {
+        try {
+          // do the actual read, from the file.
+          int bytesRead = buffer.getStream().readRemote(buffer.getOffset(), buffer.getBuffer(), 0, buffer.getRequestedLength());
+          bufferManager.doneReading(buffer, ReadBufferStatus.AVAILABLE, bytesRead);  // post result back to ReadBufferManager
+        } catch (Exception ex) {
+          bufferManager.doneReading(buffer, ReadBufferStatus.READ_FAILED, 0);
+        }
+      }
+    }
+  }
+}

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -0,0 +1,611 @@
+package seaweed.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Progressable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
+
+public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
+
+    public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
+    public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
+    public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
+    public static final String FS_SEAWEED_GRPC_CA = "fs.seaweed.ca";
+    public static final String FS_SEAWEED_GRPC_CLIENT_KEY = "fs.seaweed.client.key";
+    public static final String FS_SEAWEED_GRPC_CLIENT_CERT = "fs.seaweed.client.cert";
+
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
+    private static int BUFFER_SIZE = 16 * 1024 * 1024;
+
+    private URI uri;
+    private Path workingDirectory = new Path("/");
+    private SeaweedFileSystemStore seaweedFileSystemStore;
+
+    public URI getUri() {
+        return uri;
+    }
+
+    public String getScheme() {
+        return "seaweedfs";
+    }
+
+    @Override
+    public void initialize(URI uri, Configuration conf) throws IOException { // get
+        super.initialize(uri, conf);
+
+        // get host information from uri (overrides info in conf)
+        String host = uri.getHost();
+        host = (host == null) ? conf.get(FS_SEAWEED_FILER_HOST, "localhost") : host;
+        if (host == null) {
+            throw new IOException("Invalid host specified");
+        }
+        conf.set(FS_SEAWEED_FILER_HOST, host);
+
+        // get port information from uri, (overrides info in conf)
+        int port = uri.getPort();
+        port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port;
+        conf.setInt(FS_SEAWEED_FILER_PORT, port);
+
+        conf.setInt(IO_FILE_BUFFER_SIZE_KEY, BUFFER_SIZE);
+
+        setConf(conf);
+        this.uri = uri;
+
+        if (conf.get(FS_SEAWEED_GRPC_CA) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CA).length() != 0
+                && conf.get(FS_SEAWEED_GRPC_CLIENT_CERT) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_CERT).length() != 0
+                && conf.get(FS_SEAWEED_GRPC_CLIENT_KEY) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_KEY).length() != 0) {
+            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port,
+                    conf.get(FS_SEAWEED_GRPC_CA),
+                    conf.get(FS_SEAWEED_GRPC_CLIENT_CERT),
+                    conf.get(FS_SEAWEED_GRPC_CLIENT_KEY));
+        } else {
+            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
+        }
+
+    }
+
+    @Override
+    public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+
+        LOG.debug("open path: {} bufferSize:{}", path, bufferSize);
+
+        path = qualify(path);
+
+        try {
+            InputStream inputStream = seaweedFileSystemStore.openFileForRead(path, statistics, bufferSize);
+            return new FSDataInputStream(inputStream);
+        } catch (Exception ex) {
+            return null;
+        }
+    }
+
+    @Override
+    public FSDataOutputStream create(Path path, FsPermission permission, final boolean overwrite, final int bufferSize,
+                                     final short replication, final long blockSize, final Progressable progress) throws IOException {
+
+        LOG.debug("create path: {} bufferSize:{} blockSize:{}", path, bufferSize, blockSize);
+
+        path = qualify(path);
+
+        try {
+            String replicaPlacement = String.format("%03d", replication - 1);
+            OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, bufferSize, replicaPlacement);
+            return new FSDataOutputStream(outputStream, statistics);
+        } catch (Exception ex) {
+            return null;
+        }
+    }
+
+    @Override
+    public FSDataOutputStream append(Path path, int bufferSize, Progressable progressable) throws IOException {
+
+        LOG.debug("append path: {} bufferSize:{}", path, bufferSize);
+
+        path = qualify(path);
+        try {
+            OutputStream outputStream = seaweedFileSystemStore.createFile(path, false, null, bufferSize, "");
+            return new FSDataOutputStream(outputStream, statistics);
+        } catch (Exception ex) {
+            return null;
+        }
+    }
+
+    @Override
+    public boolean rename(Path src, Path dst) {
+
+        LOG.debug("rename path: {} => {}", src, dst);
+
+        if (src.isRoot()) {
+            return false;
+        }
+
+        if (src.equals(dst)) {
+            return true;
+        }
+        FileStatus dstFileStatus = getFileStatus(dst);
+
+        String sourceFileName = src.getName();
+        Path adjustedDst = dst;
+
+        if (dstFileStatus != null) {
+            if (!dstFileStatus.isDirectory()) {
+                return false;
+            }
+            adjustedDst = new Path(dst, sourceFileName);
+        }
+
+        Path qualifiedSrcPath = qualify(src);
+        Path qualifiedDstPath = qualify(adjustedDst);
+
+        seaweedFileSystemStore.rename(qualifiedSrcPath, qualifiedDstPath);
+        return true;
+    }
+
+    @Override
+    public boolean delete(Path path, boolean recursive) {
+
+        LOG.debug("delete path: {} recursive:{}", path, recursive);
+
+        path = qualify(path);
+
+        FileStatus fileStatus = getFileStatus(path);
+
+        if (fileStatus == null) {
+            return true;
+        }
+
+        return seaweedFileSystemStore.deleteEntries(path, fileStatus.isDirectory(), recursive);
+
+    }
+
+    @Override
+    public FileStatus[] listStatus(Path path) throws IOException {
+
+        LOG.debug("listStatus path: {}", path);
+
+        path = qualify(path);
+
+        return seaweedFileSystemStore.listEntries(path);
+    }
+
+    @Override
+    public Path getWorkingDirectory() {
+        return workingDirectory;
+    }
+
+    @Override
+    public void setWorkingDirectory(Path path) {
+        if (path.isAbsolute()) {
+            workingDirectory = path;
+        } else {
+            workingDirectory = new Path(workingDirectory, path);
+        }
+    }
+
+    @Override
+    public boolean mkdirs(Path path, FsPermission fsPermission) throws IOException {
+
+        LOG.debug("mkdirs path: {}", path);
+
+        path = qualify(path);
+
+        FileStatus fileStatus = getFileStatus(path);
+
+        if (fileStatus == null) {
+
+            UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+            return seaweedFileSystemStore.createDirectory(path, currentUser,
+                    fsPermission == null ? FsPermission.getDirDefault() : fsPermission,
+                    FsPermission.getUMask(getConf()));
+
+        }
+
+        if (fileStatus.isDirectory()) {
+            return true;
+        } else {
+            throw new FileAlreadyExistsException("Path is a file: " + path);
+        }
+    }
+
+    @Override
+    public FileStatus getFileStatus(Path path) {
+
+        LOG.debug("getFileStatus path: {}", path);
+
+        path = qualify(path);
+
+        return seaweedFileSystemStore.getFileStatus(path);
+    }
+
+    /**
+     * Set owner of a path (i.e. a file or a directory).
+     * The parameters owner and group cannot both be null.
+     *
+     * @param path  The path
+     * @param owner If it is null, the original username remains unchanged.
+     * @param group If it is null, the original groupname remains unchanged.
+     */
+    @Override
+    public void setOwner(Path path, final String owner, final String group)
+            throws IOException {
+        LOG.debug("setOwner path: {}", path);
+        path = qualify(path);
+
+        seaweedFileSystemStore.setOwner(path, owner, group);
+    }
+
+
+    /**
+     * Set permission of a path.
+     *
+     * @param path       The path
+     * @param permission Access permission
+     */
+    @Override
+    public void setPermission(Path path, final FsPermission permission) throws IOException {
+        LOG.debug("setPermission path: {}", path);
+
+        if (permission == null) {
+            throw new IllegalArgumentException("The permission can't be null");
+        }
+
+        path = qualify(path);
+
+        seaweedFileSystemStore.setPermission(path, permission);
+    }
+
+    Path qualify(Path path) {
+        return path.makeQualified(uri, workingDirectory);
+    }
+
+    /**
+     * Concat existing files together.
+     *
+     * @param trg   the path to the target destination.
+     * @param psrcs the paths to the sources to use for the concatenation.
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default).
+     */
+    @Override
+    public void concat(final Path trg, final Path[] psrcs) throws IOException {
+        throw new UnsupportedOperationException("Not implemented by the " +
+                getClass().getSimpleName() + " FileSystem implementation");
+    }
+
+    /**
+     * Truncate the file in the indicated path to the indicated size.
+     * <ul>
+     * <li>Fails if path is a directory.</li>
+     * <li>Fails if path does not exist.</li>
+     * <li>Fails if path is not closed.</li>
+     * <li>Fails if new size is greater than current size.</li>
+     * </ul>
+     *
+     * @param f         The path to the file to be truncated
+     * @param newLength The size the file is to be truncated to
+     * @return <code>true</code> if the file has been truncated to the desired
+     * <code>newLength</code> and is immediately available to be reused for
+     * write operations such as <code>append</code>, or
+     * <code>false</code> if a background process of adjusting the length of
+     * the last block has been started, and clients should wait for it to
+     * complete before proceeding with further file updates.
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default).
+     */
+    @Override
+    public boolean truncate(Path f, long newLength) throws IOException {
+        throw new UnsupportedOperationException("Not implemented by the " +
+                getClass().getSimpleName() + " FileSystem implementation");
+    }
+
+    @Override
+    public void createSymlink(final Path target, final Path link,
+                              final boolean createParent) throws AccessControlException,
+            FileAlreadyExistsException, FileNotFoundException,
+            ParentNotDirectoryException, UnsupportedFileSystemException,
+            IOException {
+        // Supporting filesystems should override this method
+        throw new UnsupportedOperationException(
+                "Filesystem does not support symlinks!");
+    }
+
+    public boolean supportsSymlinks() {
+        return false;
+    }
+
+    /**
+     * Create a snapshot.
+     *
+     * @param path         The directory where snapshots will be taken.
+     * @param snapshotName The name of the snapshot
+     * @return the snapshot path.
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     */
+    @Override
+    public Path createSnapshot(Path path, String snapshotName)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support createSnapshot");
+    }
+
+    /**
+     * Rename a snapshot.
+     *
+     * @param path            The directory path where the snapshot was taken
+     * @param snapshotOldName Old name of the snapshot
+     * @param snapshotNewName New name of the snapshot
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void renameSnapshot(Path path, String snapshotOldName,
+                               String snapshotNewName) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support renameSnapshot");
+    }
+
+    /**
+     * Delete a snapshot of a directory.
+     *
+     * @param path         The directory that the to-be-deleted snapshot belongs to
+     * @param snapshotName The name of the snapshot
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void deleteSnapshot(Path path, String snapshotName)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support deleteSnapshot");
+    }
+
+    /**
+     * Modifies ACL entries of files and directories.  This method can add new ACL
+     * entries or modify the permissions on existing ACL entries.  All existing
+     * ACL entries that are not specified in this call are retained without
+     * changes.  (Modifications are merged into the current ACL.)
+     *
+     * @param path    Path to modify
+     * @param aclSpec List&lt;AclEntry&gt; describing modifications
+     * @throws IOException                   if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support modifyAclEntries");
+    }
+
+    /**
+     * Removes ACL entries from files and directories.  Other ACL entries are
+     * retained.
+     *
+     * @param path    Path to modify
+     * @param aclSpec List describing entries to remove
+     * @throws IOException                   if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void removeAclEntries(Path path, List<AclEntry> aclSpec)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support removeAclEntries");
+    }
+
+    /**
+     * Removes all default ACL entries from files and directories.
+     *
+     * @param path Path to modify
+     * @throws IOException                   if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void removeDefaultAcl(Path path)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support removeDefaultAcl");
+    }
+
+    /**
+     * Removes all but the base ACL entries of files and directories.  The entries
+     * for user, group, and others are retained for compatibility with permission
+     * bits.
+     *
+     * @param path Path to modify
+     * @throws IOException                   if an ACL could not be removed
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void removeAcl(Path path)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support removeAcl");
+    }
+
+    /**
+     * Fully replaces ACL of files and directories, discarding all existing
+     * entries.
+     *
+     * @param path    Path to modify
+     * @param aclSpec List describing modifications, which must include entries
+     *                for user, group, and others for compatibility with permission bits.
+     * @throws IOException                   if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support setAcl");
+    }
+
+    /**
+     * Gets the ACL of a file or directory.
+     *
+     * @param path Path to get
+     * @return AclStatus describing the ACL of the file or directory
+     * @throws IOException                   if an ACL could not be read
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public AclStatus getAclStatus(Path path) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support getAclStatus");
+    }
+
+    /**
+     * Set an xattr of a file or directory.
+     * The name must be prefixed with the namespace followed by ".". For example,
+     * "user.attr".
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path  Path to modify
+     * @param name  xattr name.
+     * @param value xattr value.
+     * @param flag  xattr set flag
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void setXAttr(Path path, String name, byte[] value,
+                         EnumSet<XAttrSetFlag> flag) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support setXAttr");
+    }
+
+    /**
+     * Get an xattr name and value for a file or directory.
+     * The name must be prefixed with the namespace followed by ".". For example,
+     * "user.attr".
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attribute
+     * @param name xattr name.
+     * @return byte[] xattr value.
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public byte[] getXAttr(Path path, String name) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support getXAttr");
+    }
+
+    /**
+     * Get all of the xattr name/value pairs for a file or directory.
+     * Only those xattrs which the logged-in user has permissions to view
+     * are returned.
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attributes
+     * @return Map describing the XAttrs of the file or directory
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public Map<String, byte[]> getXAttrs(Path path) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support getXAttrs");
+    }
+
+    /**
+     * Get all of the xattrs name/value pairs for a file or directory.
+     * Only those xattrs which the logged-in user has permissions to view
+     * are returned.
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path  Path to get extended attributes
+     * @param names XAttr names.
+     * @return Map describing the XAttrs of the file or directory
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public Map<String, byte[]> getXAttrs(Path path, List<String> names)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support getXAttrs");
+    }
+
+    /**
+     * Get all of the xattr names for a file or directory.
+     * Only those xattr names which the logged-in user has permissions to view
+     * are returned.
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attributes
+     * @return List{@literal <String>} of the XAttr names of the file or directory
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public List<String> listXAttrs(Path path) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support listXAttrs");
+    }
+
+    /**
+     * Remove an xattr of a file or directory.
+     * The name must be prefixed with the namespace followed by ".". For example,
+     * "user.attr".
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to remove extended attribute
+     * @param name xattr name
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void removeXAttr(Path path, String name) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support removeXAttr");
+    }
+
+}

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -0,0 +1,277 @@
+package seaweed.hdfs;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import seaweedfs.client.FilerClient;
+import seaweedfs.client.FilerGrpcClient;
+import seaweedfs.client.FilerProto;
+import seaweedfs.client.SeaweedRead;
+
+import javax.net.ssl.SSLException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class SeaweedFileSystemStore {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystemStore.class);
+
+    private FilerGrpcClient filerGrpcClient;
+    private FilerClient filerClient;
+
+    public SeaweedFileSystemStore(String host, int port) {
+        int grpcPort = 10000 + port;
+        filerGrpcClient = new FilerGrpcClient(host, grpcPort);
+        filerClient = new FilerClient(filerGrpcClient);
+    }
+
+    public SeaweedFileSystemStore(String host, int port,
+                                  String caFile, String clientCertFile, String clientKeyFile) throws SSLException {
+        int grpcPort = 10000 + port;
+        filerGrpcClient = new FilerGrpcClient(host, grpcPort, caFile, clientCertFile, clientKeyFile);
+        filerClient = new FilerClient(filerGrpcClient);
+    }
+
+    public static String getParentDirectory(Path path) {
+        return path.isRoot() ? "/" : path.getParent().toUri().getPath();
+    }
+
+    static int permissionToMode(FsPermission permission, boolean isDirectory) {
+        int p = permission.toShort();
+        if (isDirectory) {
+            p = p | 1 << 31;
+        }
+        return p;
+    }
+
+    public boolean createDirectory(final Path path, UserGroupInformation currentUser,
+                                   final FsPermission permission, final FsPermission umask) {
+
+        LOG.debug("createDirectory path: {} permission: {} umask: {}",
+            path,
+            permission,
+            umask);
+
+        return filerClient.mkdirs(
+            path.toUri().getPath(),
+            permissionToMode(permission, true),
+            currentUser.getUserName(),
+            currentUser.getGroupNames()
+        );
+    }
+
+    public FileStatus[] listEntries(final Path path) {
+        LOG.debug("listEntries path: {}", path);
+
+        List<FileStatus> fileStatuses = new ArrayList<FileStatus>();
+
+        List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
+
+        for (FilerProto.Entry entry : entries) {
+
+            FileStatus fileStatus = doGetFileStatus(new Path(path, entry.getName()), entry);
+
+            fileStatuses.add(fileStatus);
+        }
+        return fileStatuses.toArray(new FileStatus[0]);
+    }
+
+    public FileStatus getFileStatus(final Path path) {
+
+        FilerProto.Entry entry = lookupEntry(path);
+        if (entry == null) {
+            return null;
+        }
+        LOG.debug("doGetFileStatus path:{} entry:{}", path, entry);
+
+        FileStatus fileStatus = doGetFileStatus(path, entry);
+        return fileStatus;
+    }
+
+    public boolean deleteEntries(final Path path, boolean isDirectory, boolean recursive) {
+        LOG.debug("deleteEntries path: {} isDirectory {} recursive: {}",
+            path,
+            String.valueOf(isDirectory),
+            String.valueOf(recursive));
+
+        if (path.isRoot()) {
+            return true;
+        }
+
+        if (recursive && isDirectory) {
+            List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
+            for (FilerProto.Entry entry : entries) {
+                deleteEntries(new Path(path, entry.getName()), entry.getIsDirectory(), true);
+            }
+        }
+
+        return filerClient.deleteEntry(getParentDirectory(path), path.getName(), true, recursive);
+    }
+
+    private FileStatus doGetFileStatus(Path path, FilerProto.Entry entry) {
+        FilerProto.FuseAttributes attributes = entry.getAttributes();
+        long length = SeaweedRead.totalSize(entry.getChunksList());
+        boolean isDir = entry.getIsDirectory();
+        int block_replication = 1;
+        int blocksize = 512;
+        long modification_time = attributes.getMtime() * 1000; // milliseconds
+        long access_time = 0;
+        FsPermission permission = FsPermission.createImmutable((short) attributes.getFileMode());
+        String owner = attributes.getUserName();
+        String group = attributes.getGroupNameCount() > 0 ? attributes.getGroupName(0) : "";
+        return new FileStatus(length, isDir, block_replication, blocksize,
+            modification_time, access_time, permission, owner, group, null, path);
+    }
+
+    private FilerProto.Entry lookupEntry(Path path) {
+
+        return filerClient.lookupEntry(getParentDirectory(path), path.getName());
+
+    }
+
+    public void rename(Path source, Path destination) {
+
+        LOG.debug("rename source: {} destination:{}", source, destination);
+
+        if (source.isRoot()) {
+            return;
+        }
+        LOG.warn("rename lookupEntry source: {}", source);
+        FilerProto.Entry entry = lookupEntry(source);
+        if (entry == null) {
+            LOG.warn("rename non-existing source: {}", source);
+            return;
+        }
+        filerClient.mv(source.toUri().getPath(), destination.toUri().getPath());
+    }
+
+    public OutputStream createFile(final Path path,
+                                   final boolean overwrite,
+                                   FsPermission permission,
+                                   int bufferSize,
+                                   String replication) throws IOException {
+
+        permission = permission == null ? FsPermission.getFileDefault() : permission;
+
+        LOG.debug("createFile path: {} overwrite: {} permission: {}",
+            path,
+            overwrite,
+            permission.toString());
+
+        UserGroupInformation userGroupInformation = UserGroupInformation.getCurrentUser();
+        long now = System.currentTimeMillis() / 1000L;
+
+        FilerProto.Entry.Builder entry = null;
+        long writePosition = 0;
+        if (!overwrite) {
+            FilerProto.Entry existingEntry = lookupEntry(path);
+            LOG.debug("createFile merged entry path:{} existingEntry:{}", path, existingEntry);
+            if (existingEntry != null) {
+                entry = FilerProto.Entry.newBuilder();
+                entry.mergeFrom(existingEntry);
+                entry.getAttributesBuilder().setMtime(now);
+            }
+            LOG.debug("createFile merged entry path:{} entry:{} from:{}", path, entry, existingEntry);
+            writePosition = SeaweedRead.totalSize(existingEntry.getChunksList());
+            replication = existingEntry.getAttributes().getReplication();
+        }
+        if (entry == null) {
+            entry = FilerProto.Entry.newBuilder()
+                .setName(path.getName())
+                .setIsDirectory(false)
+                .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                    .setFileMode(permissionToMode(permission, false))
+                    .setReplication(replication)
+                    .setCrtime(now)
+                    .setMtime(now)
+                    .setUserName(userGroupInformation.getUserName())
+                    .clearGroupName()
+                    .addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames()))
+                );
+        }
+
+        return new SeaweedOutputStream(filerGrpcClient, path, entry, writePosition, bufferSize, replication);
+
+    }
+
+    public InputStream openFileForRead(final Path path, FileSystem.Statistics statistics,
+                                       int bufferSize) throws IOException {
+
+        LOG.debug("openFileForRead path:{} bufferSize:{}", path, bufferSize);
+
+        int readAheadQueueDepth = 2;
+        FilerProto.Entry entry = lookupEntry(path);
+
+        if (entry == null) {
+            throw new FileNotFoundException("read non-exist file " + path);
+        }
+
+        return new SeaweedInputStream(filerGrpcClient,
+            statistics,
+            path.toUri().getPath(),
+            entry,
+            bufferSize,
+            readAheadQueueDepth);
+    }
+
+    public void setOwner(Path path, String owner, String group) {
+
+        LOG.debug("setOwner path:{} owner:{} group:{}", path, owner, group);
+
+        FilerProto.Entry entry = lookupEntry(path);
+        if (entry == null) {
+            LOG.debug("setOwner path:{} entry:{}", path, entry);
+            return;
+        }
+
+        FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
+        FilerProto.FuseAttributes.Builder attributesBuilder = entry.getAttributes().toBuilder();
+
+        if (owner != null) {
+            attributesBuilder.setUserName(owner);
+        }
+        if (group != null) {
+            attributesBuilder.clearGroupName();
+            attributesBuilder.addGroupName(group);
+        }
+
+        entryBuilder.setAttributes(attributesBuilder);
+
+        LOG.debug("setOwner path:{} entry:{}", path, entryBuilder);
+
+        filerClient.updateEntry(getParentDirectory(path), entryBuilder.build());
+
+    }
+
+    public void setPermission(Path path, FsPermission permission) {
+
+        LOG.debug("setPermission path:{} permission:{}", path, permission);
+
+        FilerProto.Entry entry = lookupEntry(path);
+        if (entry == null) {
+            LOG.debug("setPermission path:{} entry:{}", path, entry);
+            return;
+        }
+
+        FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
+        FilerProto.FuseAttributes.Builder attributesBuilder = entry.getAttributes().toBuilder();
+
+        attributesBuilder.setFileMode(permissionToMode(permission, entry.getIsDirectory()));
+
+        entryBuilder.setAttributes(attributesBuilder);
+
+        LOG.debug("setPermission path:{} entry:{}", path, entryBuilder);
+
+        filerClient.updateEntry(getParentDirectory(path), entryBuilder.build());
+
+    }
+}

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedInputStream.java

@@ -0,0 +1,371 @@
+package seaweed.hdfs;
+
+// based on org.apache.hadoop.fs.azurebfs.services.AbfsInputStream
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import seaweedfs.client.FilerGrpcClient;
+import seaweedfs.client.FilerProto;
+import seaweedfs.client.SeaweedRead;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.List;
+
+public class SeaweedInputStream extends FSInputStream {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedInputStream.class);
+
+    private final FilerGrpcClient filerGrpcClient;
+    private final Statistics statistics;
+    private final String path;
+    private final FilerProto.Entry entry;
+    private final List<SeaweedRead.VisibleInterval> visibleIntervalList;
+    private final long contentLength;
+    private final int bufferSize; // default buffer size
+    private final int readAheadQueueDepth;         // initialized in constructor
+    private final boolean readAheadEnabled; // whether enable readAhead;
+
+    private byte[] buffer = null;            // will be initialized on first use
+
+    private long fCursor = 0;  // cursor of buffer within file - offset of next byte to read from remote server
+    private long fCursorAfterLastRead = -1;
+    private int bCursor = 0;   // cursor of read within buffer - offset of next byte to be returned from buffer
+    private int limit = 0;     // offset of next byte to be read into buffer from service (i.e., upper marker+1
+    //                                                      of valid bytes in buffer)
+    private boolean closed = false;
+
+    public SeaweedInputStream(
+        final FilerGrpcClient filerGrpcClient,
+        final Statistics statistics,
+        final String path,
+        final FilerProto.Entry entry,
+        final int bufferSize,
+        final int readAheadQueueDepth) {
+        this.filerGrpcClient = filerGrpcClient;
+        this.statistics = statistics;
+        this.path = path;
+        this.entry = entry;
+        this.contentLength = SeaweedRead.totalSize(entry.getChunksList());
+        this.bufferSize = bufferSize;
+        this.readAheadQueueDepth = (readAheadQueueDepth >= 0) ? readAheadQueueDepth : Runtime.getRuntime().availableProcessors();
+        this.readAheadEnabled = true;
+
+        this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(entry.getChunksList());
+
+        LOG.debug("new path:{} entry:{} visibleIntervalList:{}", path, entry, visibleIntervalList);
+
+    }
+
+    public String getPath() {
+        return path;
+    }
+
+    @Override
+    public int read() throws IOException {
+        byte[] b = new byte[1];
+        int numberOfBytesRead = read(b, 0, 1);
+        if (numberOfBytesRead < 0) {
+            return -1;
+        } else {
+            return (b[0] & 0xFF);
+        }
+    }
+
+    @Override
+    public synchronized int read(final byte[] b, final int off, final int len) throws IOException {
+        int currentOff = off;
+        int currentLen = len;
+        int lastReadBytes;
+        int totalReadBytes = 0;
+        do {
+            lastReadBytes = readOneBlock(b, currentOff, currentLen);
+            if (lastReadBytes > 0) {
+                currentOff += lastReadBytes;
+                currentLen -= lastReadBytes;
+                totalReadBytes += lastReadBytes;
+            }
+            if (currentLen <= 0 || currentLen > b.length - currentOff) {
+                break;
+            }
+        } while (lastReadBytes > 0);
+        return totalReadBytes > 0 ? totalReadBytes : lastReadBytes;
+    }
+
+    private int readOneBlock(final byte[] b, final int off, final int len) throws IOException {
+        if (closed) {
+            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+        }
+
+        Preconditions.checkNotNull(b);
+
+        if (len == 0) {
+            return 0;
+        }
+
+        if (this.available() == 0) {
+            return -1;
+        }
+
+        if (off < 0 || len < 0 || len > b.length - off) {
+            throw new IndexOutOfBoundsException();
+        }
+
+        //If buffer is empty, then fill the buffer.
+        if (bCursor == limit) {
+            //If EOF, then return -1
+            if (fCursor >= contentLength) {
+                return -1;
+            }
+
+            long bytesRead = 0;
+            //reset buffer to initial state - i.e., throw away existing data
+            bCursor = 0;
+            limit = 0;
+            if (buffer == null) {
+                buffer = new byte[bufferSize];
+            }
+
+            // Enable readAhead when reading sequentially
+            if (-1 == fCursorAfterLastRead || fCursorAfterLastRead == fCursor || b.length >= bufferSize) {
+                bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false);
+            } else {
+                bytesRead = readInternal(fCursor, buffer, 0, b.length, true);
+            }
+
+            if (bytesRead == -1) {
+                return -1;
+            }
+
+            limit += bytesRead;
+            fCursor += bytesRead;
+            fCursorAfterLastRead = fCursor;
+        }
+
+        //If there is anything in the buffer, then return lesser of (requested bytes) and (bytes in buffer)
+        //(bytes returned may be less than requested)
+        int bytesRemaining = limit - bCursor;
+        int bytesToRead = Math.min(len, bytesRemaining);
+        System.arraycopy(buffer, bCursor, b, off, bytesToRead);
+        bCursor += bytesToRead;
+        if (statistics != null) {
+            statistics.incrementBytesRead(bytesToRead);
+        }
+        return bytesToRead;
+    }
+
+
+    private int readInternal(final long position, final byte[] b, final int offset, final int length,
+                             final boolean bypassReadAhead) throws IOException {
+        if (readAheadEnabled && !bypassReadAhead) {
+            // try reading from read-ahead
+            if (offset != 0) {
+                throw new IllegalArgumentException("readahead buffers cannot have non-zero buffer offsets");
+            }
+            int receivedBytes;
+
+            // queue read-aheads
+            int numReadAheads = this.readAheadQueueDepth;
+            long nextSize;
+            long nextOffset = position;
+            while (numReadAheads > 0 && nextOffset < contentLength) {
+                nextSize = Math.min((long) bufferSize, contentLength - nextOffset);
+                ReadBufferManager.getBufferManager().queueReadAhead(this, nextOffset, (int) nextSize);
+                nextOffset = nextOffset + nextSize;
+                numReadAheads--;
+            }
+
+            // try reading from buffers first
+            receivedBytes = ReadBufferManager.getBufferManager().getBlock(this, position, length, b);
+            if (receivedBytes > 0) {
+                return receivedBytes;
+            }
+
+            // got nothing from read-ahead, do our own read now
+            receivedBytes = readRemote(position, b, offset, length);
+            return receivedBytes;
+        } else {
+            return readRemote(position, b, offset, length);
+        }
+    }
+
+    int readRemote(long position, byte[] b, int offset, int length) throws IOException {
+        if (position < 0) {
+            throw new IllegalArgumentException("attempting to read from negative offset");
+        }
+        if (position >= contentLength) {
+            return -1;  // Hadoop prefers -1 to EOFException
+        }
+        if (b == null) {
+            throw new IllegalArgumentException("null byte array passed in to read() method");
+        }
+        if (offset >= b.length) {
+            throw new IllegalArgumentException("offset greater than length of array");
+        }
+        if (length < 0) {
+            throw new IllegalArgumentException("requested read length is less than zero");
+        }
+        if (length > (b.length - offset)) {
+            throw new IllegalArgumentException("requested read length is more than will fit after requested offset in buffer");
+        }
+
+        long bytesRead = SeaweedRead.read(filerGrpcClient, visibleIntervalList, position, b, offset, length);
+        if (bytesRead > Integer.MAX_VALUE) {
+            throw new IOException("Unexpected Content-Length");
+        }
+        return (int) bytesRead;
+    }
+
+    /**
+     * Seek to given position in stream.
+     *
+     * @param n position to seek to
+     * @throws IOException  if there is an error
+     * @throws EOFException if attempting to seek past end of file
+     */
+    @Override
+    public synchronized void seek(long n) throws IOException {
+        if (closed) {
+            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+        }
+        if (n < 0) {
+            throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
+        }
+        if (n > contentLength) {
+            throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
+        }
+
+        if (n >= fCursor - limit && n <= fCursor) { // within buffer
+            bCursor = (int) (n - (fCursor - limit));
+            return;
+        }
+
+        // next read will read from here
+        fCursor = n;
+
+        //invalidate buffer
+        limit = 0;
+        bCursor = 0;
+    }
+
+    @Override
+    public synchronized long skip(long n) throws IOException {
+        if (closed) {
+            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+        }
+        long currentPos = getPos();
+        if (currentPos == contentLength) {
+            if (n > 0) {
+                throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
+            }
+        }
+        long newPos = currentPos + n;
+        if (newPos < 0) {
+            newPos = 0;
+            n = newPos - currentPos;
+        }
+        if (newPos > contentLength) {
+            newPos = contentLength;
+            n = newPos - currentPos;
+        }
+        seek(newPos);
+        return n;
+    }
+
+    /**
+     * Return the size of the remaining available bytes
+     * if the size is less than or equal to {@link Integer#MAX_VALUE},
+     * otherwise, return {@link Integer#MAX_VALUE}.
+     * <p>
+     * This is to match the behavior of DFSInputStream.available(),
+     * which some clients may rely on (HBase write-ahead log reading in
+     * particular).
+     */
+    @Override
+    public synchronized int available() throws IOException {
+        if (closed) {
+            throw new IOException(
+                FSExceptionMessages.STREAM_IS_CLOSED);
+        }
+        final long remaining = this.contentLength - this.getPos();
+        return remaining <= Integer.MAX_VALUE
+            ? (int) remaining : Integer.MAX_VALUE;
+    }
+
+    /**
+     * Returns the length of the file that this stream refers to. Note that the length returned is the length
+     * as of the time the Stream was opened. Specifically, if there have been subsequent appends to the file,
+     * they wont be reflected in the returned length.
+     *
+     * @return length of the file.
+     * @throws IOException if the stream is closed
+     */
+    public long length() throws IOException {
+        if (closed) {
+            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+        }
+        return contentLength;
+    }
+
+    /**
+     * Return the current offset from the start of the file
+     *
+     * @throws IOException throws {@link IOException} if there is an error
+     */
+    @Override
+    public synchronized long getPos() throws IOException {
+        if (closed) {
+            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+        }
+        return fCursor - limit + bCursor;
+    }
+
+    /**
+     * Seeks a different copy of the data.  Returns true if
+     * found a new source, false otherwise.
+     *
+     * @throws IOException throws {@link IOException} if there is an error
+     */
+    @Override
+    public boolean seekToNewSource(long l) throws IOException {
+        return false;
+    }
+
+    @Override
+    public synchronized void close() throws IOException {
+        closed = true;
+        buffer = null; // de-reference the buffer so it can be GC'ed sooner
+    }
+
+    /**
+     * Not supported by this stream. Throws {@link UnsupportedOperationException}
+     *
+     * @param readlimit ignored
+     */
+    @Override
+    public synchronized void mark(int readlimit) {
+        throw new UnsupportedOperationException("mark()/reset() not supported on this stream");
+    }
+
+    /**
+     * Not supported by this stream. Throws {@link UnsupportedOperationException}
+     */
+    @Override
+    public synchronized void reset() throws IOException {
+        throw new UnsupportedOperationException("mark()/reset() not supported on this stream");
+    }
+
+    /**
+     * gets whether mark and reset are supported by {@code ADLFileInputStream}. Always returns false.
+     *
+     * @return always {@code false}
+     */
+    @Override
+    public boolean markSupported() {
+        return false;
+    }
+}

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedOutputStream.java

@@ -0,0 +1,335 @@
+package seaweed.hdfs;
+
+// adapted from org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.fs.Syncable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import seaweedfs.client.FilerGrpcClient;
+import seaweedfs.client.FilerProto;
+import seaweedfs.client.SeaweedWrite;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.io.OutputStream;
+import java.util.Locale;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import static seaweed.hdfs.SeaweedFileSystemStore.getParentDirectory;
+
+public class SeaweedOutputStream extends OutputStream implements Syncable, StreamCapabilities {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedOutputStream.class);
+
+    private final FilerGrpcClient filerGrpcClient;
+    private final Path path;
+    private final int bufferSize;
+    private final int maxConcurrentRequestCount;
+    private final ThreadPoolExecutor threadExecutor;
+    private final ExecutorCompletionService<Void> completionService;
+    private FilerProto.Entry.Builder entry;
+    private long position;
+    private boolean closed;
+    private boolean supportFlush = true;
+    private volatile IOException lastError;
+    private long lastFlushOffset;
+    private long lastTotalAppendOffset = 0;
+    private byte[] buffer;
+    private int bufferIndex;
+    private ConcurrentLinkedDeque<WriteOperation> writeOperations;
+    private String replication = "000";
+
+    public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final Path path, FilerProto.Entry.Builder entry,
+                               final long position, final int bufferSize, final String replication) {
+        this.filerGrpcClient = filerGrpcClient;
+        this.replication = replication;
+        this.path = path;
+        this.position = position;
+        this.closed = false;
+        this.lastError = null;
+        this.lastFlushOffset = 0;
+        this.bufferSize = bufferSize;
+        this.buffer = new byte[bufferSize];
+        this.bufferIndex = 0;
+        this.writeOperations = new ConcurrentLinkedDeque<>();
+
+        this.maxConcurrentRequestCount = 4 * Runtime.getRuntime().availableProcessors();
+
+        this.threadExecutor
+            = new ThreadPoolExecutor(maxConcurrentRequestCount,
+            maxConcurrentRequestCount,
+            10L,
+            TimeUnit.SECONDS,
+            new LinkedBlockingQueue<Runnable>());
+        this.completionService = new ExecutorCompletionService<>(this.threadExecutor);
+
+        this.entry = entry;
+
+    }
+
+    private synchronized void flushWrittenBytesToServiceInternal(final long offset) throws IOException {
+
+        LOG.debug("SeaweedWrite.writeMeta path: {} entry:{}", path, entry);
+
+        try {
+            SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry);
+        } catch (Exception ex) {
+            throw new IOException(ex);
+        }
+        this.lastFlushOffset = offset;
+    }
+
+    @Override
+    public void write(final int byteVal) throws IOException {
+        write(new byte[]{(byte) (byteVal & 0xFF)});
+    }
+
+    @Override
+    public synchronized void write(final byte[] data, final int off, final int length)
+        throws IOException {
+        maybeThrowLastError();
+
+        Preconditions.checkArgument(data != null, "null data");
+
+        if (off < 0 || length < 0 || length > data.length - off) {
+            throw new IndexOutOfBoundsException();
+        }
+
+        int currentOffset = off;
+        int writableBytes = bufferSize - bufferIndex;
+        int numberOfBytesToWrite = length;
+
+        while (numberOfBytesToWrite > 0) {
+            if (writableBytes <= numberOfBytesToWrite) {
+                System.arraycopy(data, currentOffset, buffer, bufferIndex, writableBytes);
+                bufferIndex += writableBytes;
+                writeCurrentBufferToService();
+                currentOffset += writableBytes;
+                numberOfBytesToWrite = numberOfBytesToWrite - writableBytes;
+            } else {
+                System.arraycopy(data, currentOffset, buffer, bufferIndex, numberOfBytesToWrite);
+                bufferIndex += numberOfBytesToWrite;
+                numberOfBytesToWrite = 0;
+            }
+
+            writableBytes = bufferSize - bufferIndex;
+        }
+    }
+
+    /**
+     * Flushes this output stream and forces any buffered output bytes to be
+     * written out. If any data remains in the payload it is committed to the
+     * service. Data is queued for writing and forced out to the service
+     * before the call returns.
+     */
+    @Override
+    public void flush() throws IOException {
+        if (supportFlush) {
+            flushInternalAsync();
+        }
+    }
+
+    /**
+     * Similar to posix fsync, flush out the data in client's user buffer
+     * all the way to the disk device (but the disk may have it in its cache).
+     *
+     * @throws IOException if error occurs
+     */
+    @Override
+    public void hsync() throws IOException {
+        if (supportFlush) {
+            flushInternal();
+        }
+    }
+
+    /**
+     * Flush out the data in client's user buffer. After the return of
+     * this call, new readers will see the data.
+     *
+     * @throws IOException if any error occurs
+     */
+    @Override
+    public void hflush() throws IOException {
+        if (supportFlush) {
+            flushInternal();
+        }
+    }
+
+    /**
+     * Query the stream for a specific capability.
+     *
+     * @param capability string to query the stream support for.
+     * @return true for hsync and hflush.
+     */
+    @Override
+    public boolean hasCapability(String capability) {
+        switch (capability.toLowerCase(Locale.ENGLISH)) {
+            case StreamCapabilities.HSYNC:
+            case StreamCapabilities.HFLUSH:
+                return supportFlush;
+            default:
+                return false;
+        }
+    }
+
+    /**
+     * Force all data in the output stream to be written to Azure storage.
+     * Wait to return until this is complete. Close the access to the stream and
+     * shutdown the upload thread pool.
+     * If the blob was created, its lease will be released.
+     * Any error encountered caught in threads and stored will be rethrown here
+     * after cleanup.
+     */
+    @Override
+    public synchronized void close() throws IOException {
+        if (closed) {
+            return;
+        }
+
+        LOG.debug("close path: {}", path);
+        try {
+            flushInternal();
+            threadExecutor.shutdown();
+        } finally {
+            lastError = new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+            buffer = null;
+            bufferIndex = 0;
+            closed = true;
+            writeOperations.clear();
+            if (!threadExecutor.isShutdown()) {
+                threadExecutor.shutdownNow();
+            }
+        }
+    }
+
+    private synchronized void writeCurrentBufferToService() throws IOException {
+        if (bufferIndex == 0) {
+            return;
+        }
+
+        final byte[] bytes = buffer;
+        final int bytesLength = bufferIndex;
+
+        buffer = new byte[bufferSize];
+        bufferIndex = 0;
+        final long offset = position;
+        position += bytesLength;
+
+        if (threadExecutor.getQueue().size() >= maxConcurrentRequestCount * 2) {
+            waitForTaskToComplete();
+        }
+
+        final Future<Void> job = completionService.submit(new Callable<Void>() {
+            @Override
+            public Void call() throws Exception {
+                // originally: client.append(path, offset, bytes, 0, bytesLength);
+                SeaweedWrite.writeData(entry, replication, filerGrpcClient, offset, bytes, 0, bytesLength);
+                return null;
+            }
+        });
+
+        writeOperations.add(new WriteOperation(job, offset, bytesLength));
+
+        // Try to shrink the queue
+        shrinkWriteOperationQueue();
+    }
+
+    private void waitForTaskToComplete() throws IOException {
+        boolean completed;
+        for (completed = false; completionService.poll() != null; completed = true) {
+            // keep polling until there is no data
+        }
+
+        if (!completed) {
+            try {
+                completionService.take();
+            } catch (InterruptedException e) {
+                lastError = (IOException) new InterruptedIOException(e.toString()).initCause(e);
+                throw lastError;
+            }
+        }
+    }
+
+    private void maybeThrowLastError() throws IOException {
+        if (lastError != null) {
+            throw lastError;
+        }
+    }
+
+    /**
+     * Try to remove the completed write operations from the beginning of write
+     * operation FIFO queue.
+     */
+    private synchronized void shrinkWriteOperationQueue() throws IOException {
+        try {
+            while (writeOperations.peek() != null && writeOperations.peek().task.isDone()) {
+                writeOperations.peek().task.get();
+                lastTotalAppendOffset += writeOperations.peek().length;
+                writeOperations.remove();
+            }
+        } catch (Exception e) {
+            lastError = new IOException(e);
+            throw lastError;
+        }
+    }
+
+    private synchronized void flushInternal() throws IOException {
+        maybeThrowLastError();
+        writeCurrentBufferToService();
+        flushWrittenBytesToService();
+    }
+
+    private synchronized void flushInternalAsync() throws IOException {
+        maybeThrowLastError();
+        writeCurrentBufferToService();
+        flushWrittenBytesToServiceAsync();
+    }
+
+    private synchronized void flushWrittenBytesToService() throws IOException {
+        for (WriteOperation writeOperation : writeOperations) {
+            try {
+                writeOperation.task.get();
+            } catch (Exception ex) {
+                lastError = new IOException(ex);
+                throw lastError;
+            }
+        }
+        LOG.debug("flushWrittenBytesToService: {} position:{}", path, position);
+        flushWrittenBytesToServiceInternal(position);
+    }
+
+    private synchronized void flushWrittenBytesToServiceAsync() throws IOException {
+        shrinkWriteOperationQueue();
+
+        if (this.lastTotalAppendOffset > this.lastFlushOffset) {
+            this.flushWrittenBytesToServiceInternal(this.lastTotalAppendOffset);
+        }
+    }
+
+    private static class WriteOperation {
+        private final Future<Void> task;
+        private final long startOffset;
+        private final long length;
+
+        WriteOperation(final Future<Void> task, final long startOffset, final long length) {
+            Preconditions.checkNotNull(task, "task");
+            Preconditions.checkArgument(startOffset >= 0, "startOffset");
+            Preconditions.checkArgument(length >= 0, "length");
+
+            this.task = task;
+            this.startOffset = startOffset;
+            this.length = length;
+        }
+    }
+
+}

other/java/s3copier/pom.xml

@@ -0,0 +1,35 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.seaweedfs.s3</groupId>
+  <artifactId>copier</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0-SNAPSHOT</version>
+  <name>copier</name>
+  <url>http://maven.apache.org</url>
+
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+        <groupId>com.amazonaws</groupId>
+        <artifactId>aws-java-sdk-bom</artifactId>
+        <version>1.11.327</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>aws-java-sdk-s3</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>3.8.1</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>

other/java/s3copier/src/main/java/com/seaweedfs/s3/HighLevelMultipartUpload.java

@@ -0,0 +1,63 @@
+package com.seaweedfs.s3;
+
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.SdkClientException;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
+import com.amazonaws.services.s3.transfer.Upload;
+
+import java.io.File;
+
+public class HighLevelMultipartUpload {
+
+    public static void main(String[] args) throws Exception {
+        String bucketName = "javabucket";
+        String filePath = args[0];
+        File file = new File(filePath);
+        String keyName = "path/to/" + file.getName();
+
+        try {
+            AWSCredentials credentials = new BasicAWSCredentials("ANY-ACCESSKEYID", "ANY-SECRETACCESSKEY");
+            ClientConfiguration clientConfiguration = new ClientConfiguration();
+            clientConfiguration.setSignerOverride("AWSS3V4SignerType");
+
+            AmazonS3 s3Client = AmazonS3ClientBuilder
+                .standard()
+                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
+                    "http://localhost:8333", Regions.US_WEST_1.name()))
+                .withPathStyleAccessEnabled(true)
+                .withClientConfiguration(clientConfiguration)
+                .withCredentials(new AWSStaticCredentialsProvider(credentials))
+                .build();
+
+            TransferManager tm = TransferManagerBuilder.standard()
+                .withS3Client(s3Client)
+                .build();
+
+            // TransferManager processes all transfers asynchronously,
+            // so this call returns immediately.
+            Upload upload = tm.upload(bucketName, keyName, file);
+            System.out.println("Object upload started");
+
+            // Optionally, wait for the upload to finish before continuing.
+            upload.waitForCompletion();
+            System.out.println("Object upload complete");
+        } catch (AmazonServiceException e) {
+            // The call was transmitted successfully, but Amazon S3 couldn't process
+            // it, so it returned an error response.
+            e.printStackTrace();
+        } catch (SdkClientException e) {
+            // Amazon S3 couldn't be contacted for a response, or the client
+            // couldn't parse the response from Amazon S3.
+            e.printStackTrace();
+        }
+    }
+}

other/java/s3copier/src/main/java/com/seaweedfs/s3/PutObject.java

@@ -0,0 +1,101 @@
+package com.seaweedfs.s3;
+
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.SdkClientException;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.util.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Hello world!
+ */
+public class PutObject {
+
+    private static Log log = LogFactory.getLog(PutObject.class);
+
+    public static void main(String[] args) {
+
+        AWSCredentials credentials = new BasicAWSCredentials("ANY-ACCESSKEYID", "ANY-SECRETACCESSKEY");
+        ClientConfiguration clientConfiguration = new ClientConfiguration();
+        clientConfiguration.setSignerOverride("AWSS3V4SignerType");
+
+        AmazonS3 s3Client = AmazonS3ClientBuilder
+            .standard()
+            .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
+                "http://localhost:8333", Regions.US_WEST_1.name()))
+            .withPathStyleAccessEnabled(true)
+            .withClientConfiguration(clientConfiguration)
+            .withCredentials(new AWSStaticCredentialsProvider(credentials))
+            .build();
+
+        String bucketName = "javabucket";
+        String stringObjKeyName = "strObject2";
+        String fileObjKeyName = "fileObject2";
+        String fileName = args[0];
+
+        String stringContent = "Uploaded String Object v3";
+        try {
+
+            // Upload a text string as a new object.
+            s3Client.putObject(bucketName, stringObjKeyName, stringContent);
+
+            // Upload a file as a new object with ContentType and title specified.
+            PutObjectRequest request = new PutObjectRequest(bucketName, fileObjKeyName, new File(fileName));
+            ObjectMetadata metadata = new ObjectMetadata();
+            metadata.setContentType("plain/text");
+            metadata.addUserMetadata("x-amz-meta-title", "someTitle");
+            request.setMetadata(metadata);
+            s3Client.putObject(request);
+
+            // test reads
+            S3Object written = s3Client.getObject(bucketName, stringObjKeyName);
+            try {
+                String expected = IOUtils.toString(written.getObjectContent());
+
+                if (!stringContent.equals(expected)){
+                    System.out.println("Failed to put and get back the content!");
+                }
+
+            } catch (IOException e) {
+                throw new SdkClientException("Error streaming content from S3 during download");
+            } finally {
+                IOUtils.closeQuietly(written, log);
+            }
+
+            // test deletes
+            s3Client.deleteObject(bucketName, stringObjKeyName);
+
+
+            // delete bucket
+            String tmpBucket = "tmpbucket";
+            s3Client.createBucket(tmpBucket);
+            s3Client.putObject(tmpBucket, stringObjKeyName, stringContent);
+            s3Client.deleteBucket(tmpBucket);
+
+        } catch (AmazonServiceException e) {
+            // The call was transmitted successfully, but Amazon S3 couldn't process
+            // it, so it returned an error response.
+            e.printStackTrace();
+        } catch (SdkClientException e) {
+            // Amazon S3 couldn't be contacted for a response, or the client
+            // couldn't parse the response from Amazon S3.
+            e.printStackTrace();
+        }
+
+        System.out.println("Hello World!");
+    }
+}

other/java/s3copier/src/test/java/com/seaweedfs/s3/PutObjectTest.java

@@ -0,0 +1,38 @@
+package com.seaweedfs.s3;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * Unit test for simple App.
+ */
+public class PutObjectTest
+    extends TestCase
+{
+    /**
+     * Create the test case
+     *
+     * @param testName name of the test case
+     */
+    public PutObjectTest(String testName )
+    {
+        super( testName );
+    }
+
+    /**
+     * @return the suite of tests being tested
+     */
+    public static Test suite()
+    {
+        return new TestSuite( PutObjectTest.class );
+    }
+
+    /**
+     * Rigourous Test :-)
+     */
+    public void testApp()
+    {
+        assertTrue( true );
+    }
+}

unmaintained/change_superblock/change_superblock.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 )
 
 var (
@@ -48,11 +49,7 @@ func main() {
 	}
 	defer datFile.Close()
 
-	header := make([]byte, storage.SuperBlockSize)
-	if _, e := datFile.Read(header); e != nil {
-		glog.Fatalf("cannot read volume %s super block: %v", fileName+".dat", e)
-	}
-	superBlock, err := storage.ParseSuperBlock(header)
+	superBlock, err := storage.ReadSuperBlock(datFile)
 
 	if err != nil {
 		glog.Fatalf("cannot parse existing super block: %v", err)
@@ -77,7 +74,7 @@ func main() {
 	}
 
 	if *targetTTL != "" {
-		ttl, err := storage.ReadTTL(*targetTTL)
+		ttl, err := needle.ReadTTL(*targetTTL)
 
 		if err != nil {
 			glog.Fatalf("cannot parse target ttl %s: %v", *targetTTL, err)
@@ -91,7 +88,7 @@ func main() {
 
 	if hasChange {
 
-		header = superBlock.Bytes()
+		header := superBlock.Bytes()
 
 		if n, e := datFile.WriteAt(header, 0); n == 0 || e != nil {
 			glog.Fatalf("cannot write super block: %v", e)

unmaintained/fix_dat/fix_dat.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -39,34 +41,36 @@ func main() {
 	}
 	indexFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
 	if err != nil {
-		glog.Fatalf("Read Volume Index [ERROR] %s\n", err)
+		glog.Fatalf("Read Volume Index %v", err)
 	}
 	defer indexFile.Close()
 	datFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".dat"), os.O_RDONLY, 0644)
 	if err != nil {
-		glog.Fatalf("Read Volume Data [ERROR] %s\n", err)
+		glog.Fatalf("Read Volume Data %v", err)
 	}
 	defer datFile.Close()
 
 	newDatFile, err := os.Create(path.Join(*fixVolumePath, fileName+".dat_fixed"))
 	if err != nil {
-		glog.Fatalf("Write New Volume Data [ERROR] %s\n", err)
+		glog.Fatalf("Write New Volume Data %v", err)
 	}
 	defer newDatFile.Close()
 
-	header := make([]byte, storage.SuperBlockSize)
-	datFile.Read(header)
-	newDatFile.Write(header)
+	superBlock, err := storage.ReadSuperBlock(datFile)
+	if err != nil {
+		glog.Fatalf("Read Volume Data superblock %v", err)
+	}
+	newDatFile.Write(superBlock.Bytes())
 
-	iterateEntries(datFile, indexFile, func(n *storage.Needle, offset int64) {
-		fmt.Printf("file id=%d name=%s size=%d dataSize=%d\n", n.Id, string(n.Name), n.Size, n.DataSize)
-		s, _, e := n.Append(newDatFile, storage.Version2)
+	iterateEntries(datFile, indexFile, func(n *needle.Needle, offset int64) {
+		fmt.Printf("needle id=%v name=%s size=%d dataSize=%d\n", n.Id, string(n.Name), n.Size, n.DataSize)
+		_, s, _, e := n.Append(newDatFile, superBlock.Version())
 		fmt.Printf("size %d error %v\n", s, e)
 	})
 
 }
 
-func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needle, offset int64)) {
+func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *needle.Needle, offset int64)) {
 	// start to read index file
 	var readerOffset int64
 	bytes := make([]byte, 16)
@@ -74,9 +78,14 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 	readerOffset += int64(count)
 
 	// start to read dat file
-	offset := int64(storage.SuperBlockSize)
-	version := storage.Version2
-	n, rest, err := storage.ReadNeedleHeader(datFile, version, offset)
+	superBlock, err := storage.ReadSuperBlock(datFile)
+	if err != nil {
+		fmt.Printf("cannot read dat file super block: %v", err)
+		return
+	}
+	offset := int64(superBlock.BlockSize())
+	version := superBlock.Version()
+	n, _, rest, err := needle.ReadNeedleHeader(datFile, version, offset)
 	if err != nil {
 		fmt.Printf("cannot read needle header: %v", err)
 		return
@@ -98,8 +107,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 
 		fmt.Printf("key: %d offsetFromIndex %d n.Size %d sizeFromIndex:%d\n", key, offsetFromIndex, n.Size, sizeFromIndex)
 
-		padding := storage.NeedlePaddingSize - ((sizeFromIndex + storage.NeedleHeaderSize + storage.NeedleChecksumSize) % storage.NeedlePaddingSize)
-		rest = sizeFromIndex + storage.NeedleChecksumSize + padding
+		rest = needle.NeedleBodyLength(sizeFromIndex, version)
 
 		func() {
 			defer func() {
@@ -107,7 +115,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 					fmt.Println("Recovered in f", r)
 				}
 			}()
-			if err = n.ReadNeedleBody(datFile, version, offset+int64(storage.NeedleHeaderSize), rest); err != nil {
+			if _, err = n.ReadNeedleBody(datFile, version, offset+int64(types.NeedleHeaderSize), rest); err != nil {
 				fmt.Printf("cannot read needle body: offset %d body %d %v\n", offset, rest, err)
 			}
 		}()
@@ -117,9 +125,9 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 		}
 		visitNeedle(n, offset)
 
-		offset += int64(storage.NeedleHeaderSize) + int64(rest)
+		offset += types.NeedleHeaderSize + rest
 		//fmt.Printf("==> new entry offset %d\n", offset)
-		if n, rest, err = storage.ReadNeedleHeader(datFile, version, offset); err != nil {
+		if n, _, rest, err = needle.ReadNeedleHeader(datFile, version, offset); err != nil {
 			if err == io.EOF {
 				return
 			}

unmaintained/repeated_vacuum/repeated_vacuum.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
+	"log"
+	"math/rand"
+
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+var (
+	master = flag.String("master", "127.0.0.1:9333", "the master server")
+	repeat = flag.Int("n", 5, "repeat how many times")
+)
+
+func main() {
+	flag.Parse()
+
+	weed_server.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
+	for i := 0; i < *repeat; i++ {
+		assignResult, err := operation.Assign(*master, grpcDialOption, &operation.VolumeAssignRequest{Count: 1})
+		if err != nil {
+			log.Fatalf("assign: %v", err)
+		}
+
+		data := make([]byte, 1024)
+		rand.Read(data)
+		reader := bytes.NewReader(data)
+
+		targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
+
+		_, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), reader, false, "", nil, assignResult.Auth)
+		if err != nil {
+			log.Fatalf("upload: %v", err)
+		}
+
+		util.Delete(targetUrl, string(assignResult.Auth))
+
+		util.Get(fmt.Sprintf("http://%s/vol/vacuum", *master))
+
+	}
+
+}

unmaintained/see_dat/see_dat.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+
+	"time"
+)
+
+var (
+	volumePath       = flag.String("dir", "/tmp", "data directory to store files")
+	volumeCollection = flag.String("collection", "", "the volume collection name")
+	volumeId         = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
+)
+
+type VolumeFileScanner4SeeDat struct {
+	version needle.Version
+}
+
+func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock storage.SuperBlock) error {
+	scanner.version = superBlock.Version()
+	return nil
+
+}
+func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
+	return true
+}
+
+func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64) error {
+	t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
+	glog.V(0).Infof("%d,%s%x offset %d size %d cookie %x appendedAt %v", *volumeId, n.Id, n.Cookie, offset, n.Size, n.Cookie, t)
+	return nil
+}
+
+func main() {
+	flag.Parse()
+
+	vid := needle.VolumeId(*volumeId)
+
+	scanner := &VolumeFileScanner4SeeDat{}
+	err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)
+	if err != nil {
+		glog.Fatalf("Reading Volume File [ERROR] %s\n", err)
+	}
+
+}

unmaintained/see_idx/see_idx.go

@@ -2,12 +2,14 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"os"
 	"path"
 	"strconv"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 
 var (
@@ -33,6 +35,9 @@ func main() {
 	}
 	defer indexFile.Close()
 
-	storage.LoadNeedleMap(indexFile)
+	storage.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
+		fmt.Printf("key:%v offset:%v size:%v\n", key, offset, size)
+		return nil
+	})
 
 }

unmaintained/see_meta/see_meta.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/golang/protobuf/proto"
+)
+
+var (
+	metaFile = flag.String("meta", "", "meta file generated via fs.meta.save")
+)
+
+func main() {
+	flag.Parse()
+
+	dst, err := os.OpenFile(*metaFile, os.O_RDONLY, 0644)
+	if err != nil {
+		log.Fatalf("failed to open %s: %v", *metaFile, err)
+	}
+	defer dst.Close()
+
+	err = walkMetaFile(dst)
+	if err != nil {
+		log.Fatalf("failed to visit %s: %v", *metaFile, err)
+	}
+
+}
+
+func walkMetaFile(dst *os.File) error {
+
+	sizeBuf := make([]byte, 4)
+
+	for {
+		if n, err := dst.Read(sizeBuf); n != 4 {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+
+		size := util.BytesToUint32(sizeBuf)
+
+		data := make([]byte, int(size))
+
+		if n, err := dst.Read(data); n != len(data) {
+			return err
+		}
+
+		fullEntry := &filer_pb.FullEntry{}
+		if err := proto.Unmarshal(data, fullEntry); err != nil {
+			return err
+		}
+
+		fmt.Fprintf(os.Stdout, "file %s %v\n", filer2.FullPath(fullEntry.Dir).Child(fullEntry.Entry.Name), fullEntry.Entry.Attributes.String())
+		for i, chunk := range fullEntry.Entry.Chunks {
+			fmt.Fprintf(os.Stdout, "  chunk %d %v\n", i+1, chunk.String())
+		}
+
+	}
+
+}

unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	size        = flag.Int("size", 1024, "file size")
+	concurrency = flag.Int("c", 4, "concurrent number of uploads")
+	times       = flag.Int("n", 1024, "repeated number of times")
+	fileCount   = flag.Int("fileCount", 1, "number of files to write")
+	destination = flag.String("to", "http://localhost:8888/", "destination directory on filer")
+
+	statsChan = make(chan stat, 8)
+)
+
+type stat struct {
+	size int64
+}
+
+func main() {
+
+	flag.Parse()
+
+	data := make([]byte, *size)
+	println("data len", len(data))
+
+	var wg sync.WaitGroup
+	for x := 0; x < *concurrency; x++ {
+		wg.Add(1)
+
+		go func(x int) {
+			defer wg.Done()
+
+			client := &http.Client{Transport: &http.Transport{
+				MaxConnsPerHost:     1024,
+				MaxIdleConnsPerHost: 1024,
+			}}
+			r := rand.New(rand.NewSource(time.Now().UnixNano() + int64(x)))
+
+			for t := 0; t < *times; t++ {
+				for f := 0; f < *fileCount; f++ {
+					fn := r.Intn(*fileCount)
+					if size, err := uploadFileToFiler(client, data, fmt.Sprintf("file%04d", fn), *destination); err == nil {
+						statsChan <- stat{
+							size: size,
+						}
+					} else {
+						log.Fatalf("client %d upload %d times: %v", x, t, err)
+					}
+				}
+			}
+		}(x)
+	}
+
+	go func() {
+		ticker := time.NewTicker(1000 * time.Millisecond)
+
+		var lastTime time.Time
+		var counter, size int64
+		for {
+			select {
+			case stat := <-statsChan:
+				size += stat.size
+				counter++
+			case x := <-ticker.C:
+				if !lastTime.IsZero() {
+					elapsed := x.Sub(lastTime).Seconds()
+					fmt.Fprintf(os.Stdout, "%.2f files/s, %.2f MB/s\n",
+						float64(counter)/elapsed,
+						float64(size/1024/1024)/elapsed)
+				}
+				lastTime = x
+				size = 0
+				counter = 0
+			}
+		}
+	}()
+
+	wg.Wait()
+
+}
+
+func uploadFileToFiler(client *http.Client, data []byte, filename, destination string) (size int64, err error) {
+
+	if !strings.HasSuffix(destination, "/") {
+		destination = destination + "/"
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", filename)
+	if err != nil {
+		return 0, fmt.Errorf("fail to create form %v: %v", filename, err)
+	}
+
+	part.Write(data)
+
+	err = writer.Close()
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", filename, err)
+	}
+
+	uri := destination + filename
+
+	request, err := http.NewRequest("POST", uri, body)
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	// request.Close = true  // can not use this, which do not reuse http connection, impacting filer->volume also.
+
+	resp, err := client.Do(request)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	} else {
+		body := &bytes.Buffer{}
+		_, err := body.ReadFrom(resp.Body)
+		if err != nil {
+			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
+		}
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+	}
+
+	return int64(len(data)), nil
+}

unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go

@@ -0,0 +1,150 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	dir         = flag.String("dir", ".", "upload files under this directory")
+	concurrency = flag.Int("c", 1, "concurrent number of uploads")
+	times       = flag.Int("n", 1, "repeated number of times")
+	destination = flag.String("to", "http://localhost:8888/", "destination directory on filer")
+
+	statsChan = make(chan stat, 8)
+)
+
+type stat struct {
+	size int64
+}
+
+func main() {
+
+	flag.Parse()
+
+	var fileNames []string
+
+	files, err := ioutil.ReadDir(*dir)
+	if err != nil {
+		log.Fatalf("fail to read dir %v: %v", *dir, err)
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		fileNames = append(fileNames, filepath.Join(*dir, file.Name()))
+	}
+
+	var wg sync.WaitGroup
+	for x := 0; x < *concurrency; x++ {
+		wg.Add(1)
+
+		client := &http.Client{}
+
+		go func() {
+			defer wg.Done()
+			rand.Shuffle(len(fileNames), func(i, j int) {
+				fileNames[i], fileNames[j] = fileNames[j], fileNames[i]
+			})
+			for t := 0; t < *times; t++ {
+				for _, filename := range fileNames {
+					if size, err := uploadFileToFiler(client, filename, *destination); err == nil {
+						statsChan <- stat{
+							size: size,
+						}
+					}
+				}
+			}
+		}()
+	}
+
+	go func() {
+		ticker := time.NewTicker(500 * time.Millisecond)
+
+		var lastTime time.Time
+		var counter, size int64
+		for {
+			select {
+			case stat := <-statsChan:
+				size += stat.size
+				counter++
+			case x := <-ticker.C:
+				if !lastTime.IsZero() {
+					elapsed := x.Sub(lastTime).Seconds()
+					fmt.Fprintf(os.Stdout, "%.2f files/s, %.2f MB/s\n",
+						float64(counter)/elapsed,
+						float64(size/1024/1024)/elapsed)
+				}
+				lastTime = x
+				size = 0
+				counter = 0
+			}
+		}
+	}()
+
+	wg.Wait()
+
+}
+
+func uploadFileToFiler(client *http.Client, filename, destination string) (size int64, err error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		panic(err)
+	}
+	defer file.Close()
+
+	fi, err := file.Stat()
+
+	if !strings.HasSuffix(destination, "/") {
+		destination = destination + "/"
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", file.Name())
+	if err != nil {
+		return 0, fmt.Errorf("fail to create form %v: %v", file.Name(), err)
+	}
+	_, err = io.Copy(part, file)
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", file.Name(), err)
+	}
+
+	err = writer.Close()
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", file.Name(), err)
+	}
+
+	uri := destination + file.Name()
+
+	request, err := http.NewRequest("POST", uri, body)
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+
+	resp, err := client.Do(request)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	} else {
+		body := &bytes.Buffer{}
+		_, err := body.ReadFrom(resp.Body)
+		if err != nil {
+			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
+		}
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+	}
+
+	return fi.Size(), nil
+}

unmaintained/volume_tailer/volume_tailer.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	weed_server "github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	util2 "github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/spf13/viper"
+	"golang.org/x/tools/godoc/util"
+)
+
+var (
+	master         = flag.String("master", "localhost:9333", "master server host and port")
+	volumeId       = flag.Int("volumeId", -1, "a volume id")
+	rewindDuration = flag.Duration("rewind", -1, "rewind back in time. -1 means from the first entry. 0 means from now.")
+	timeoutSeconds = flag.Int("timeoutSeconds", 0, "disconnect if no activity after these seconds")
+	showTextFile   = flag.Bool("showTextFile", false, "display textual file content")
+)
+
+func main() {
+	flag.Parse()
+
+	weed_server.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
+	vid := needle.VolumeId(*volumeId)
+
+	var sinceTimeNs int64
+	if *rewindDuration == 0 {
+		sinceTimeNs = time.Now().UnixNano()
+	} else if *rewindDuration == -1 {
+		sinceTimeNs = 0
+	} else if *rewindDuration > 0 {
+		sinceTimeNs = time.Now().Add(-*rewindDuration).UnixNano()
+	}
+
+	err := operation.TailVolume(*master, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
+		if n.Size == 0 {
+			println("-", n.String())
+			return nil
+		} else {
+			println("+", n.String())
+		}
+
+		if *showTextFile {
+
+			data := n.Data
+			if n.IsGzipped() {
+				if data, err = util2.UnGzipData(data); err != nil {
+					return err
+				}
+			}
+			if util.IsText(data) {
+				println(string(data))
+			}
+
+			println("-", n.String(), "compressed", n.IsGzipped(), "original size", len(data))
+		}
+		return nil
+	})
+
+	if err != nil {
+		log.Printf("Error VolumeTailSender volume %d: %v", vid, err)
+	}
+
+}

util/gostd

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+############################  GLOBAL VARIABLES
+regex=' '
+branch="master"
+max_length=150
+
+REGEX_SUFFIX_GO=".+\.go$"
+
+############################  FUNCTIONS
+msg() {
+    printf '%b' "$1" >&2
+}
+
+die() {
+    msg "\33[31m[✘]\33[0m ${1}${2}"
+    exit 1
+}
+
+succ() {
+    msg "\33[34m[√]\33[0m ${1}${2}"
+}
+
+gostd() {
+    local branch=$1
+    local reg4exclude=$2
+    local max_length=$3
+
+    for file in `git diff $branch --name-only`
+    do
+        if ! [[ $file =~ $REGEX_SUFFIX_GO ]] || [[ $file =~ $reg4exclude ]]; then
+            continue
+        fi
+
+        error=`go fmt $file 2>&1`
+        if ! [ $? -eq 0 ]; then
+            die "go fmt $file:" "$error"
+        fi
+
+        succ "$file\n"
+
+        grep -n -E --color=always ".{$max_length}" $file | awk '{ printf ("%4s %s\n", "", $0) }'
+    done
+}
+
+get_options() {
+    while getopts "b:e:hl:" opts
+    do
+        case $opts in
+            b)
+                branch=$OPTARG
+                ;;
+            e)
+                regex=$OPTARG
+                ;;
+            h)
+                usage
+                exit 0
+                ;;
+            l)
+                max_length=$OPTARG
+                ;;
+            \?)
+                usage
+                exit 1
+                ;;
+        esac
+    done
+}
+
+usage () {
+    cat << _EOC_
+Usage:
+    gostd [options]
+
+Options:
+    -b <branch/commit>        Specify the git diff branch or commit.
+                              (default: master)
+    -e <regex>                Regex for excluding file or directory.
+    -h                        Print this usage.
+    -l <length>               Show files that exceed the limit line length.
+                              (default: 150)
+
+Examples:
+    gostd
+    gostd -b master -l 100
+    gostd -b 59d532a -e weed/pb -l 100
+_EOC_
+}
+
+main() {
+    get_options "$@"
+
+    gostd "$branch" "$regex" "$max_length"
+}
+
+############################  MAIN()
+main "$@"

weed/command/backup.go

@@ -3,6 +3,11 @@ package command
 import (
 	"fmt"
 
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/spf13/viper"
+
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 )
@@ -30,26 +35,30 @@ var cmdBackup = &Command{
 	UsageLine: "backup -dir=. -volumeId=234 -server=localhost:9333",
 	Short:     "incrementally backup a volume to local folder",
 	Long: `Incrementally backup volume data.
-	
+
 	It is expected that you use this inside a script, to loop through
 	all possible volume ids that needs to be backup to local folder.
-	
+
 	The volume id does not need to exist locally or even remotely.
 	This will help to backup future new volumes.
-	
+
 	Usually backing up is just copying the .dat (and .idx) files.
-	But it's tricky to incremententally copy the differences.
-	
+	But it's tricky to incrementally copy the differences.
+
 	The complexity comes when there are multiple addition, deletion and compaction.
-	This tool will handle them correctly and efficiently, avoiding unnecessary data transporation.
+	This tool will handle them correctly and efficiently, avoiding unnecessary data transportation.
   `,
 }
 
 func runBackup(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
 	if *s.volumeId == -1 {
 		return false
 	}
-	vid := storage.VolumeId(*s.volumeId)
+	vid := needle.VolumeId(*s.volumeId)
 
 	// find volume location, replication, ttl info
 	lookup, err := operation.Lookup(*s.master, vid.String())
@@ -59,12 +68,12 @@ func runBackup(cmd *Command, args []string) bool {
 	}
 	volumeServer := lookup.Locations[0].Url
 
-	stats, err := operation.GetVolumeSyncStatus(volumeServer, vid.String())
+	stats, err := operation.GetVolumeSyncStatus(volumeServer, grpcDialOption, uint32(vid))
 	if err != nil {
 		fmt.Printf("Error get volume %d status: %v\n", vid, err)
 		return true
 	}
-	ttl, err := storage.ReadTTL(stats.Ttl)
+	ttl, err := needle.ReadTTL(stats.Ttl)
 	if err != nil {
 		fmt.Printf("Error get volume %d ttl %s: %v\n", vid, stats.Ttl, err)
 		return true
@@ -81,7 +90,34 @@ func runBackup(cmd *Command, args []string) bool {
 		return true
 	}
 
-	if err := v.Synchronize(volumeServer); err != nil {
+	if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
+		if err = v.Compact(0, 0); err != nil {
+			fmt.Printf("Compact Volume before synchronizing %v\n", err)
+			return true
+		}
+		if err = v.CommitCompact(); err != nil {
+			fmt.Printf("Commit Compact before synchronizing %v\n", err)
+			return true
+		}
+		v.SuperBlock.CompactionRevision = uint16(stats.CompactRevision)
+		v.DataFile().WriteAt(v.SuperBlock.Bytes(), 0)
+	}
+
+	datSize, _, _ := v.FileStat()
+
+	if datSize > stats.TailOffset {
+		// remove the old data
+		v.Destroy()
+		// recreate an empty volume
+		v, err = storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0)
+		if err != nil {
+			fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
+			return true
+		}
+	}
+	defer v.Close()
+
+	if err := v.IncrementalBackup(volumeServer, grpcDialOption); err != nil {
 		fmt.Printf("Error synchronizing volume %d: %v\n", vid, err)
 		return true
 	}

weed/command/benchmark.go

@@ -2,7 +2,11 @@ package command
 
 import (
 	"bufio"
+	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
+	"google.golang.org/grpc"
 	"io"
 	"math"
 	"math/rand"
@@ -18,10 +22,11 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )
 
 type BenchmarkOptions struct {
-	server           *string
+	masters          *string
 	concurrency      *int
 	numberOfFiles    *int
 	fileSize         *int
@@ -31,20 +36,23 @@ type BenchmarkOptions struct {
 	read             *bool
 	sequentialRead   *bool
 	collection       *string
+	replication      *string
 	cpuprofile       *string
 	maxCpu           *int
-	secretKey        *string
+	grpcDialOption   grpc.DialOption
+	masterClient     *wdclient.MasterClient
 }
 
 var (
 	b           BenchmarkOptions
 	sharedBytes []byte
+	isSecure    bool
 )
 
 func init() {
-	cmdBenchmark.Run = runbenchmark // break init cycle
+	cmdBenchmark.Run = runBenchmark // break init cycle
 	cmdBenchmark.IsDebug = cmdBenchmark.Flag.Bool("debug", false, "verbose debug information")
-	b.server = cmdBenchmark.Flag.String("server", "localhost:9333", "SeaweedFS master location")
+	b.masters = cmdBenchmark.Flag.String("master", "localhost:9333", "SeaweedFS master location")
 	b.concurrency = cmdBenchmark.Flag.Int("c", 16, "number of concurrent write or read processes")
 	b.fileSize = cmdBenchmark.Flag.Int("size", 1024, "simulated file size in bytes, with random(0~63) bytes padding")
 	b.numberOfFiles = cmdBenchmark.Flag.Int("n", 1024*1024, "number of files to write for each thread")
@@ -54,9 +62,9 @@ func init() {
 	b.read = cmdBenchmark.Flag.Bool("read", true, "enable read")
 	b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file")
 	b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection")
+	b.replication = cmdBenchmark.Flag.String("replication", "000", "replication type")
 	b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
 	b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
-	b.secretKey = cmdBenchmark.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
 	sharedBytes = make([]byte, 1024)
 }
 
@@ -98,7 +106,11 @@ var (
 	readStats  *stats
 )
 
-func runbenchmark(cmd *Command, args []string) bool {
+func runBenchmark(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("security", false)
+	b.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
+
 	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
 	if *b.maxCpu < 1 {
 		*b.maxCpu = runtime.NumCPU()
@@ -113,18 +125,22 @@ func runbenchmark(cmd *Command, args []string) bool {
 		defer pprof.StopCPUProfile()
 	}
 
+	b.masterClient = wdclient.NewMasterClient(context.Background(), b.grpcDialOption, "client", strings.Split(*b.masters, ","))
+	go b.masterClient.KeepConnectedToMaster()
+	b.masterClient.WaitUntilConnected()
+
 	if *b.write {
-		bench_write()
+		benchWrite()
 	}
 
 	if *b.read {
-		bench_read()
+		benchRead()
 	}
 
 	return true
 }
 
-func bench_write() {
+func benchWrite() {
 	fileIdLineChan := make(chan string)
 	finishChan := make(chan bool)
 	writeStats = newStats(*b.concurrency)
@@ -151,7 +167,7 @@ func bench_write() {
 	writeStats.printStats()
 }
 
-func bench_read() {
+func benchRead() {
 	fileIdLineChan := make(chan string)
 	finishChan := make(chan bool)
 	readStats = newStats(*b.concurrency)
@@ -181,7 +197,6 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 	defer wait.Done()
 	delayedDeleteChan := make(chan *delayedFile, 100)
 	var waitForDeletions sync.WaitGroup
-	secret := security.Secret(*b.secretKey)
 
 	for i := 0; i < 7; i++ {
 		waitForDeletions.Add(1)
@@ -191,8 +206,11 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 				if df.enterTime.After(time.Now()) {
 					time.Sleep(df.enterTime.Sub(time.Now()))
 				}
-				if e := util.Delete("http://"+df.fp.Server+"/"+df.fp.Fid,
-					security.GenJwt(secret, df.fp.Fid)); e == nil {
+				var jwtAuthorization security.EncodedJwt
+				if isSecure {
+					jwtAuthorization = operation.LookupJwt(b.masterClient.GetMaster(), df.fp.Fid)
+				}
+				if e := util.Delete(fmt.Sprintf("http://%s/%s", df.fp.Server, df.fp.Fid), string(jwtAuthorization)); e == nil {
 					s.completed++
 				} else {
 					s.failed++
@@ -201,18 +219,28 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 		}()
 	}
 
+	random := rand.New(rand.NewSource(time.Now().UnixNano()))
+
 	for id := range idChan {
 		start := time.Now()
-		fileSize := int64(*b.fileSize + rand.Intn(64))
-		fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize}
-		ar := &operation.VolumeAssignRequest{
-			Count:      1,
-			Collection: *b.collection,
+		fileSize := int64(*b.fileSize + random.Intn(64))
+		fp := &operation.FilePart{
+			Reader:   &FakeReader{id: uint64(id), size: fileSize},
+			FileSize: fileSize,
+			MimeType: "image/bench", // prevent gzip benchmark content
 		}
-		if assignResult, err := operation.Assign(*b.server, ar); err == nil {
+		ar := &operation.VolumeAssignRequest{
+			Count:       1,
+			Collection:  *b.collection,
+			Replication: *b.replication,
+		}
+		if assignResult, err := operation.Assign(b.masterClient.GetMaster(), b.grpcDialOption, ar); err == nil {
 			fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection
-			if _, err := fp.Upload(0, *b.server, secret); err == nil {
-				if rand.Intn(100) < *b.deletePercentage {
+			if !isSecure && assignResult.Auth != "" {
+				isSecure = true
+			}
+			if _, err := fp.Upload(0, b.masterClient.GetMaster(), assignResult.Auth, b.grpcDialOption); err == nil {
+				if random.Intn(100) < *b.deletePercentage {
 					s.total++
 					delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}
 				} else {
@@ -239,6 +267,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 
 func readFiles(fileIdLineChan chan string, s *stat) {
 	defer wait.Done()
+
 	for fid := range fileIdLineChan {
 		if len(fid) == 0 {
 			continue
@@ -249,17 +278,13 @@ func readFiles(fileIdLineChan chan string, s *stat) {
 		if *cmdBenchmark.IsDebug {
 			fmt.Printf("reading file %s\n", fid)
 		}
-		parts := strings.SplitN(fid, ",", 2)
-		vid := parts[0]
 		start := time.Now()
-		ret, err := operation.Lookup(*b.server, vid)
-		if err != nil || len(ret.Locations) == 0 {
+		url, err := b.masterClient.LookupFileId(fid)
+		if err != nil {
 			s.failed++
-			println("!!!! volume id ", vid, " location not found!!!!!")
+			println("!!!! ", fid, " location not found!!!!!")
 			continue
 		}
-		server := ret.Locations[rand.Intn(len(ret.Locations))].Url
-		url := "http://" + server + "/" + fid
 		if bytesRead, err := util.Get(url); err == nil {
 			s.completed++
 			s.transferred += int64(len(bytesRead))
@@ -297,6 +322,8 @@ func readFileIds(fileName string, fileIdLineChan chan string) {
 	}
 	defer file.Close()
 
+	random := rand.New(rand.NewSource(time.Now().UnixNano()))
+
 	r := bufio.NewReader(file)
 	if *b.sequentialRead {
 		for {
@@ -317,7 +344,7 @@ func readFileIds(fileName string, fileIdLineChan chan string) {
 		}
 		if len(lines) > 0 {
 			for i := 0; i < readStats.total; i++ {
-				fileIdLineChan <- lines[rand.Intn(len(lines))]
+				fileIdLineChan <- lines[random.Intn(len(lines))]
 			}
 		}
 	}

weed/command/command.go

@@ -13,16 +13,20 @@ var Commands = []*Command{
 	cmdCompact,
 	cmdCopy,
 	cmdFix,
+	cmdFilerReplicate,
 	cmdServer,
 	cmdMaster,
 	cmdFiler,
+	cmdS3,
 	cmdUpload,
 	cmdDownload,
+	cmdScaffold,
 	cmdShell,
 	cmdVersion,
 	cmdVolume,
 	cmdExport,
 	cmdMount,
+	cmdWebDav,
 }
 
 type Command struct {

weed/command/compact.go

@@ -3,6 +3,7 @@ package command
 import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 )
 
 func init() {
@@ -33,14 +34,16 @@ func runCompact(cmd *Command, args []string) bool {
 		return false
 	}
 
-	vid := storage.VolumeId(*compactVolumeId)
+	preallocate := *compactVolumePreallocate * (1 << 20)
+
+	vid := needle.VolumeId(*compactVolumeId)
 	v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid,
-		storage.NeedleMapInMemory, nil, nil, *compactVolumePreallocate*(1<<20))
+		storage.NeedleMapInMemory, nil, nil, preallocate)
 	if err != nil {
 		glog.Fatalf("Load Volume [ERROR] %s\n", err)
 	}
 	if *compactMethod == 0 {
-		if err = v.Compact(); err != nil {
+		if err = v.Compact(preallocate, 0); err != nil {
 			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	} else {

weed/command/download.go

@@ -55,7 +55,7 @@ func downloadToFile(server, fileId, saveDir string) error {
 	if lookupError != nil {
 		return lookupError
 	}
-	filename, rc, err := util.DownloadUrl(fileUrl)
+	filename, _, rc, err := util.DownloadFile(fileUrl)
 	if err != nil {
 		return err
 	}
@@ -108,7 +108,7 @@ func fetchContent(server string, fileId string) (filename string, content []byte
 		return "", nil, lookupError
 	}
 	var rc io.ReadCloser
-	if filename, rc, e = util.DownloadUrl(fileUrl); e != nil {
+	if filename, _, rc, e = util.DownloadFile(fileUrl); e != nil {
 		return "", nil, e
 	}
 	content, e = ioutil.ReadAll(rc)

weed/command/export.go

@@ -12,8 +12,12 @@ import (
 	"text/template"
 	"time"
 
+	"io"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 
 const (
@@ -49,9 +53,11 @@ func init() {
 }
 
 var (
-	output = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
-	format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename formatted with {{.Mime}} {{.Id}} {{.Name}} {{.Ext}}")
-	newer  = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone, e.g. 2006-01-02T15:04:05")
+	output      = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
+	format      = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename formatted with {{.Mime}} {{.Id}} {{.Name}} {{.Ext}}")
+	newer       = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone, e.g. 2006-01-02T15:04:05")
+	showDeleted = cmdExport.Flag.Bool("deleted", false, "export deleted files. only applies if -o is not specified")
+	limit       = cmdExport.Flag.Int("limit", 0, "only show first n entries if specified")
 
 	tarOutputFile          *tar.Writer
 	tarHeader              tar.Header
@@ -62,6 +68,80 @@ var (
 	localLocation, _             = time.LoadLocation("Local")
 )
 
+func printNeedle(vid needle.VolumeId, n *needle.Needle, version needle.Version, deleted bool) {
+	key := needle.NewFileIdFromNeedle(vid, n).String()
+	size := n.DataSize
+	if version == needle.Version1 {
+		size = n.Size
+	}
+	fmt.Printf("%s\t%s\t%d\t%t\t%s\t%s\t%s\t%t\n",
+		key,
+		n.Name,
+		size,
+		n.IsGzipped(),
+		n.Mime,
+		n.LastModifiedString(),
+		n.Ttl.String(),
+		deleted,
+	)
+}
+
+type VolumeFileScanner4Export struct {
+	version   needle.Version
+	counter   int
+	needleMap *storage.NeedleMap
+	vid       needle.VolumeId
+}
+
+func (scanner *VolumeFileScanner4Export) VisitSuperBlock(superBlock storage.SuperBlock) error {
+	scanner.version = superBlock.Version()
+	return nil
+
+}
+func (scanner *VolumeFileScanner4Export) ReadNeedleBody() bool {
+	return true
+}
+
+func (scanner *VolumeFileScanner4Export) VisitNeedle(n *needle.Needle, offset int64) error {
+	needleMap := scanner.needleMap
+	vid := scanner.vid
+
+	nv, ok := needleMap.Get(n.Id)
+	glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
+		n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped(), ok, nv)
+	if ok && nv.Size > 0 && nv.Size != types.TombstoneFileSize && nv.Offset.ToAcutalOffset() == offset {
+		if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
+			glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
+				n.LastModified, newerThanUnix)
+			return nil
+		}
+		scanner.counter++
+		if *limit > 0 && scanner.counter > *limit {
+			return io.EOF
+		}
+		if tarOutputFile != nil {
+			return writeFile(vid, n)
+		} else {
+			printNeedle(vid, n, scanner.version, false)
+			return nil
+		}
+	}
+	if !ok {
+		if *showDeleted && tarOutputFile == nil {
+			if n.DataSize > 0 {
+				printNeedle(vid, n, scanner.version, true)
+			} else {
+				n.Name = []byte("*tombstone")
+				printNeedle(vid, n, scanner.version, true)
+			}
+		}
+		glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
+	} else {
+		glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
+	}
+	return nil
+}
+
 func runExport(cmd *Command, args []string) bool {
 
 	var err error
@@ -111,45 +191,29 @@ func runExport(cmd *Command, args []string) bool {
 	if *export.collection != "" {
 		fileName = *export.collection + "_" + fileName
 	}
-	vid := storage.VolumeId(*export.volumeId)
+	vid := needle.VolumeId(*export.volumeId)
 	indexFile, err := os.OpenFile(path.Join(*export.dir, fileName+".idx"), os.O_RDONLY, 0644)
 	if err != nil {
 		glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
 	}
 	defer indexFile.Close()
 
-	needleMap, err := storage.LoadNeedleMap(indexFile)
+	needleMap, err := storage.LoadBtreeNeedleMap(indexFile)
 	if err != nil {
 		glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
 	}
 
-	var version storage.Version
+	volumeFileScanner := &VolumeFileScanner4Export{
+		needleMap: needleMap,
+		vid:       vid,
+	}
 
-	err = storage.ScanVolumeFile(*export.dir, *export.collection, vid,
-		storage.NeedleMapInMemory,
-		func(superBlock storage.SuperBlock) error {
-			version = superBlock.Version()
-			return nil
-		}, true, func(n *storage.Needle, offset int64) error {
-			nv, ok := needleMap.Get(n.Id)
-			glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
-				n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped(), ok, nv)
-			if ok && nv.Size > 0 && int64(nv.Offset)*8 == offset {
-				if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
-					glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
-						n.LastModified, newerThanUnix)
-					return nil
-				}
-				return walker(vid, n, version)
-			}
-			if !ok {
-				glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
-			} else {
-				glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
-			}
-			return nil
-		})
-	if err != nil {
+	if tarOutputFile == nil {
+		fmt.Printf("key\tname\tsize\tgzip\tmime\tmodified\tttl\tdeleted\n")
+	}
+
+	err = storage.ScanVolumeFile(*export.dir, *export.collection, vid, storage.NeedleMapInMemory, volumeFileScanner)
+	if err != nil && err != io.EOF {
 		glog.Fatalf("Export Volume File [ERROR] %s\n", err)
 	}
 	return true
@@ -157,57 +221,43 @@ func runExport(cmd *Command, args []string) bool {
 
 type nameParams struct {
 	Name string
-	Id   uint64
+	Id   types.NeedleId
 	Mime string
 	Key  string
 	Ext  string
 }
 
-func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) {
-	key := storage.NewFileIdFromNeedle(vid, n).String()
-	if tarOutputFile != nil {
-		fileNameTemplateBuffer.Reset()
-		if err = fileNameTemplate.Execute(fileNameTemplateBuffer,
-			nameParams{
-				Name: string(n.Name),
-				Id:   n.Id,
-				Mime: string(n.Mime),
-				Key:  key,
-				Ext:  filepath.Ext(string(n.Name)),
-			},
-		); err != nil {
-			return err
-		}
-
-		fileName := fileNameTemplateBuffer.String()
-
-		if n.IsGzipped() && path.Ext(fileName) != ".gz" {
-			fileName = fileName + ".gz"
-		}
-
-		tarHeader.Name, tarHeader.Size = fileName, int64(len(n.Data))
-		if n.HasLastModifiedDate() {
-			tarHeader.ModTime = time.Unix(int64(n.LastModified), 0)
-		} else {
-			tarHeader.ModTime = time.Unix(0, 0)
-		}
-		tarHeader.ChangeTime = tarHeader.ModTime
-		if err = tarOutputFile.WriteHeader(&tarHeader); err != nil {
-			return err
-		}
-		_, err = tarOutputFile.Write(n.Data)
-	} else {
-		size := n.DataSize
-		if version == storage.Version1 {
-			size = n.Size
-		}
-		fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n",
-			key,
-			n.Name,
-			size,
-			n.IsGzipped(),
-			n.Mime,
-		)
+func writeFile(vid needle.VolumeId, n *needle.Needle) (err error) {
+	key := needle.NewFileIdFromNeedle(vid, n).String()
+	fileNameTemplateBuffer.Reset()
+	if err = fileNameTemplate.Execute(fileNameTemplateBuffer,
+		nameParams{
+			Name: string(n.Name),
+			Id:   n.Id,
+			Mime: string(n.Mime),
+			Key:  key,
+			Ext:  filepath.Ext(string(n.Name)),
+		},
+	); err != nil {
+		return err
 	}
+
+	fileName := fileNameTemplateBuffer.String()
+
+	if n.IsGzipped() && path.Ext(fileName) != ".gz" {
+		fileName = fileName + ".gz"
+	}
+
+	tarHeader.Name, tarHeader.Size = fileName, int64(len(n.Data))
+	if n.HasLastModifiedDate() {
+		tarHeader.ModTime = time.Unix(int64(n.LastModified), 0)
+	} else {
+		tarHeader.ModTime = time.Unix(0, 0)
+	}
+	tarHeader.ChangeTime = tarHeader.ModTime
+	if err = tarOutputFile.WriteHeader(&tarHeader); err != nil {
+		return err
+	}
+	_, err = tarOutputFile.Write(n.Data)
 	return
 }

weed/command/filer.go

@@ -1,14 +1,18 @@
 package command
 
 import (
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
 	"net/http"
-	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc/reflection"
 )
 
 var (
@@ -16,48 +20,43 @@ var (
 )
 
 type FilerOptions struct {
-	master                  *string
+	masters                 *string
 	ip                      *string
 	port                    *int
+	publicPort              *int
 	collection              *string
 	defaultReplicaPlacement *string
-	dir                     *string
 	redirectOnRead          *bool
 	disableDirListing       *bool
-	confFile                *string
 	maxMB                   *int
-	secretKey               *string
-	cassandra_server        *string
-	cassandra_keyspace      *string
-	redis_server            *string
-	redis_password          *string
-	redis_database          *int
+	dirListingLimit         *int
+	dataCenter              *string
+	enableNotification      *bool
+	disableHttp             *bool
+
+	// default leveldb directory, used in "weed server" mode
+	defaultLevelDbDirectory *string
 }
 
 func init() {
 	cmdFiler.Run = runFiler // break init cycle
-	f.master = cmdFiler.Flag.String("master", "localhost:9333", "master server location")
+	f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
 	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection")
 	f.ip = cmdFiler.Flag.String("ip", "", "filer server http listen ip address")
 	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
-	f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data")
+	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
 	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
 	f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
-	f.confFile = cmdFiler.Flag.String("confFile", "", "json encoded filer conf file")
-	f.maxMB = cmdFiler.Flag.Int("maxMB", 0, "split files larger than the limit")
-	f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server")
-	f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server")
-	f.redis_server = cmdFiler.Flag.String("redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379")
-	f.redis_password = cmdFiler.Flag.String("redis.password", "", "password in clear text")
-	f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server")
-	f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
-
+	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
+	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
+	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center")
+	f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
 }
 
 var cmdFiler = &Command{
-	UsageLine: "filer -port=8888 -dir=/tmp -master=<ip:port>",
-	Short:     "start a file server that points to a master server",
+	UsageLine: "filer -port=8888 -master=<ip:port>[,<ip:port>]*",
+	Short:     "start a file server that points to a master server, or a list of master servers",
 	Long: `start a file server which accepts REST operation for any files.
 
 	//create or overwrite the file, the directories /path/to will be automatically created
@@ -69,43 +68,89 @@ var cmdFiler = &Command{
 	//return a json format subdirectory and files listing
 	GET /path/to/
 
-  Current <fullpath~fileid> mapping metadata store is local embedded leveldb.
-  It should be highly scalable to hundreds of millions of files on a modest machine.
+	The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.
 
-  Future we will ensure it can avoid of being SPOF.
+	The example filer.toml configuration file can be generated by "weed scaffold -config=filer"
 
-  `,
+`,
 }
 
 func runFiler(cmd *Command, args []string) bool {
 
-	if err := util.TestFolderWritable(*f.dir); err != nil {
-		glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err)
+	weed_server.LoadConfiguration("security", false)
+
+	f.startFiler()
+
+	return true
+}
+
+func (fo *FilerOptions) startFiler() {
+
+	defaultMux := http.NewServeMux()
+	publicVolumeMux := defaultMux
+
+	if *fo.publicPort != 0 {
+		publicVolumeMux = http.NewServeMux()
 	}
 
-	r := http.NewServeMux()
-	_, nfs_err := weed_server.NewFilerServer(r, *f.ip, *f.port, *f.master, *f.dir, *f.collection,
-		*f.defaultReplicaPlacement, *f.redirectOnRead, *f.disableDirListing,
-		*f.confFile,
-		*f.maxMB,
-		*f.secretKey,
-		*f.cassandra_server, *f.cassandra_keyspace,
-		*f.redis_server, *f.redis_password, *f.redis_database,
-	)
+	defaultLevelDbDirectory := "./filerdb"
+	if fo.defaultLevelDbDirectory != nil {
+		defaultLevelDbDirectory = *fo.defaultLevelDbDirectory + "/filerdb"
+	}
+
+	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
+		Masters:            strings.Split(*fo.masters, ","),
+		Collection:         *fo.collection,
+		DefaultReplication: *fo.defaultReplicaPlacement,
+		RedirectOnRead:     *fo.redirectOnRead,
+		DisableDirListing:  *fo.disableDirListing,
+		MaxMB:              *fo.maxMB,
+		DirListingLimit:    *fo.dirListingLimit,
+		DataCenter:         *fo.dataCenter,
+		DefaultLevelDbDir:  defaultLevelDbDirectory,
+		DisableHttp:        *fo.disableHttp,
+	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)
 	}
-	glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*f.port))
+
+	if *fo.publicPort != 0 {
+		publicListeningAddress := *fo.ip + ":" + strconv.Itoa(*fo.publicPort)
+		glog.V(0).Infoln("Start Seaweed filer server", util.VERSION, "public at", publicListeningAddress)
+		publicListener, e := util.NewListener(publicListeningAddress, 0)
+		if e != nil {
+			glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
+		}
+		go func() {
+			if e := http.Serve(publicListener, publicVolumeMux); e != nil {
+				glog.Fatalf("Volume server fail to serve public: %v", e)
+			}
+		}()
+	}
+
+	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.VERSION, *fo.ip, *fo.port)
 	filerListener, e := util.NewListener(
-		":"+strconv.Itoa(*f.port),
+		*fo.ip+":"+strconv.Itoa(*fo.port),
 		time.Duration(10)*time.Second,
 	)
 	if e != nil {
 		glog.Fatalf("Filer listener error: %v", e)
 	}
-	if e := http.Serve(filerListener, r); e != nil {
+
+	// starting grpc server
+	grpcPort := *fo.port + 10000
+	grpcL, err := util.NewListener(":"+strconv.Itoa(grpcPort), 0)
+	if err != nil {
+		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
+	}
+	grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "filer"))
+	filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
+	reflection.Register(grpcS)
+	go grpcS.Serve(grpcL)
+
+	httpS := &http.Server{Handler: defaultMux}
+	if err := httpS.Serve(filerListener); err != nil {
 		glog.Fatalf("Filer Fail to serve: %v", e)
 	}
 
-	return true
 }

weed/command/filer_copy.go

@@ -1,32 +1,45 @@
 package command
 
 import (
+	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
+	"github.com/spf13/viper"
+	"google.golang.org/grpc"
+	"io"
 	"io/ioutil"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
-
-	"github.com/chrislusf/seaweedfs/weed/operation"
-	filer_operation "github.com/chrislusf/seaweedfs/weed/operation/filer"
-	"github.com/chrislusf/seaweedfs/weed/security"
+	"sync"
+	"time"
 )
 
 var (
-	copy CopyOptions
+	copy      CopyOptions
+	waitGroup sync.WaitGroup
 )
 
 type CopyOptions struct {
-	master      *string
-	include     *string
-	replication *string
-	collection  *string
-	ttl         *string
-	maxMB       *int
-	secretKey   *string
-
-	secret security.Secret
+	filerGrpcPort    *int
+	master           *string
+	include          *string
+	replication      *string
+	collection       *string
+	ttl              *string
+	maxMB            *int
+	grpcDialOption   grpc.DialOption
+	masterClient     *wdclient.MasterClient
+	concurrency      *int
+	compressionLevel *int
 }
 
 func init() {
@@ -37,8 +50,10 @@ func init() {
 	copy.replication = cmdCopy.Flag.String("replication", "", "replication type")
 	copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
 	copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
-	copy.maxMB = cmdCopy.Flag.Int("maxMB", 0, "split files larger than the limit")
-	copy.secretKey = cmdCopy.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+	copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit")
+	copy.filerGrpcPort = cmdCopy.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to filer port + 10000")
+	copy.concurrency = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
+	copy.compressionLevel = cmdCopy.Flag.Int("compressionLevel", 9, "local file compression level 1 ~ 9")
 }
 
 var cmdCopy = &Command{
@@ -52,18 +67,15 @@ var cmdCopy = &Command{
   All files under the folder and subfolders will be copyed.
   Optional parameter "-include" allows you to specify the file name patterns.
 
-  If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
-  This can save volume server's gzipped processing and allow customizable gzip compression level.
-  The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
-
-  If "maxMB" is set to a positive number, files larger than it would be split into chunks and copyed separatedly.
-  The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
+  If "maxMB" is set to a positive number, files larger than it would be split into chunks.
 
   `,
 }
 
 func runCopy(cmd *Command, args []string) bool {
-	copy.secret = security.Secret(*copy.secretKey)
+
+	weed_server.LoadConfiguration("security", false)
+
 	if len(args) <= 1 {
 		return false
 	}
@@ -75,73 +87,335 @@ func runCopy(cmd *Command, args []string) bool {
 		fmt.Printf("The last argument should be a URL on filer: %v\n", err)
 		return false
 	}
-	path := filerUrl.Path
-	if !strings.HasSuffix(path, "/") {
-		path = path + "/"
+	urlPath := filerUrl.Path
+	if !strings.HasSuffix(urlPath, "/") {
+		fmt.Printf("The last argument should be a folder and end with \"/\": %v\n", err)
+		return false
 	}
 
-	for _, fileOrDir := range fileOrDirs {
-		if !doEachCopy(fileOrDir, filerUrl.Host, path) {
-			return false
-		}
+	if filerUrl.Port() == "" {
+		fmt.Printf("The filer port should be specified.\n")
+		return false
 	}
+
+	filerPort, parseErr := strconv.ParseUint(filerUrl.Port(), 10, 64)
+	if parseErr != nil {
+		fmt.Printf("The filer port parse error: %v\n", parseErr)
+		return false
+	}
+
+	filerGrpcPort := filerPort + 10000
+	if *copy.filerGrpcPort != 0 {
+		filerGrpcPort = uint64(*copy.filerGrpcPort)
+	}
+
+	filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort)
+	copy.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
+
+	copy.masterClient = wdclient.NewMasterClient(context.Background(), copy.grpcDialOption, "client", strings.Split(*copy.master, ","))
+	go copy.masterClient.KeepConnectedToMaster()
+	copy.masterClient.WaitUntilConnected()
+
+	if *cmdCopy.IsDebug {
+		util.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof")
+	}
+
+	fileCopyTaskChan := make(chan FileCopyTask, *copy.concurrency)
+
+	ctx := context.Background()
+
+	go func() {
+		defer close(fileCopyTaskChan)
+		for _, fileOrDir := range fileOrDirs {
+			if err := genFileCopyTask(fileOrDir, urlPath, fileCopyTaskChan); err != nil {
+				fmt.Fprintf(os.Stderr, "gen file list error: %v\n", err)
+				break
+			}
+		}
+	}()
+	for i := 0; i < *copy.concurrency; i++ {
+		waitGroup.Add(1)
+		go func() {
+			defer waitGroup.Done()
+			worker := FileCopyWorker{
+				options:          &copy,
+				filerHost:        filerUrl.Host,
+				filerGrpcAddress: filerGrpcAddress,
+			}
+			if err := worker.copyFiles(ctx, fileCopyTaskChan); err != nil {
+				fmt.Fprintf(os.Stderr, "copy file error: %v\n", err)
+				return
+			}
+		}()
+	}
+	waitGroup.Wait()
+
 	return true
 }
 
-func doEachCopy(fileOrDir string, host string, path string) bool {
-	f, err := os.Open(fileOrDir)
-	if err != nil {
-		fmt.Printf("Failed to open file %s: %v", fileOrDir, err)
-		return false
-	}
-	defer f.Close()
+func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan FileCopyTask) error {
 
-	fi, err := f.Stat()
+	fi, err := os.Stat(fileOrDir)
 	if err != nil {
-		fmt.Printf("Failed to get stat for file %s: %v", fileOrDir, err)
-		return false
+		fmt.Fprintf(os.Stderr, "Failed to get stat for file %s: %v\n", fileOrDir, err)
+		return nil
 	}
 
 	mode := fi.Mode()
 	if mode.IsDir() {
 		files, _ := ioutil.ReadDir(fileOrDir)
 		for _, subFileOrDir := range files {
-			if !doEachCopy(fileOrDir+"/"+subFileOrDir.Name(), host, path+fi.Name()+"/") {
-				return false
+			if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), destPath+fi.Name()+"/", fileCopyTaskChan); err != nil {
+				return err
 			}
 		}
-		return true
+		return nil
 	}
 
+	uid, gid := util.GetFileUidGid(fi)
+
+	fileCopyTaskChan <- FileCopyTask{
+		sourceLocation:     fileOrDir,
+		destinationUrlPath: destPath,
+		fileSize:           fi.Size(),
+		fileMode:           fi.Mode(),
+		uid:                uid,
+		gid:                gid,
+	}
+
+	return nil
+}
+
+type FileCopyWorker struct {
+	options          *CopyOptions
+	filerHost        string
+	filerGrpcAddress string
+}
+
+func (worker *FileCopyWorker) copyFiles(ctx context.Context, fileCopyTaskChan chan FileCopyTask) error {
+	for task := range fileCopyTaskChan {
+		if err := worker.doEachCopy(ctx, task); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type FileCopyTask struct {
+	sourceLocation     string
+	destinationUrlPath string
+	fileSize           int64
+	fileMode           os.FileMode
+	uid                uint32
+	gid                uint32
+}
+
+func (worker *FileCopyWorker) doEachCopy(ctx context.Context, task FileCopyTask) error {
+
+	f, err := os.Open(task.sourceLocation)
+	if err != nil {
+		fmt.Printf("Failed to open file %s: %v\n", task.sourceLocation, err)
+		if _, ok := err.(*os.PathError); ok {
+			fmt.Printf("skipping %s\n", task.sourceLocation)
+			return nil
+		}
+		return err
+	}
+	defer f.Close()
+
 	// this is a regular file
-	if *copy.include != "" {
-		if ok, _ := filepath.Match(*copy.include, filepath.Base(fileOrDir)); !ok {
-			return true
+	if *worker.options.include != "" {
+		if ok, _ := filepath.Match(*worker.options.include, filepath.Base(task.sourceLocation)); !ok {
+			return nil
 		}
 	}
 
-	parts, err := operation.NewFileParts([]string{fileOrDir})
-	if err != nil {
-		fmt.Printf("Failed to read file %s: %v", fileOrDir, err)
+	// find the chunk count
+	chunkSize := int64(*worker.options.maxMB * 1024 * 1024)
+	chunkCount := 1
+	if chunkSize > 0 && task.fileSize > chunkSize {
+		chunkCount = int(task.fileSize/chunkSize) + 1
 	}
 
-	results, err := operation.SubmitFiles(*copy.master, parts,
-		*copy.replication, *copy.collection, "",
-		*copy.ttl, *copy.maxMB, copy.secret)
-	if err != nil {
-		fmt.Printf("Failed to submit file %s: %v", fileOrDir, err)
+	if chunkCount == 1 {
+		return worker.uploadFileAsOne(ctx, task, f)
 	}
 
-	if strings.HasSuffix(path, "/") {
-		path = path + fi.Name()
+	return worker.uploadFileInChunks(ctx, task, f, chunkCount, chunkSize)
+}
+
+func (worker *FileCopyWorker) uploadFileAsOne(ctx context.Context, task FileCopyTask, f *os.File) error {
+
+	// upload the file content
+	fileName := filepath.Base(f.Name())
+	mimeType := detectMimeType(f)
+
+	var chunks []*filer_pb.FileChunk
+
+	if task.fileSize > 0 {
+
+		// assign a volume
+		assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
+			Count:       1,
+			Replication: *worker.options.replication,
+			Collection:  *worker.options.collection,
+			Ttl:         *worker.options.ttl,
+		})
+		if err != nil {
+			fmt.Printf("Failed to assign from %s: %v\n", *worker.options.master, err)
+		}
+
+		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
+
+		uploadResult, err := operation.UploadWithLocalCompressionLevel(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth, *worker.options.compressionLevel)
+		if err != nil {
+			return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
+		}
+		if uploadResult.Error != "" {
+			return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
+		}
+		fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
+
+		chunks = append(chunks, &filer_pb.FileChunk{
+			FileId: assignResult.Fid,
+			Offset: 0,
+			Size:   uint64(uploadResult.Size),
+			Mtime:  time.Now().UnixNano(),
+			ETag:   uploadResult.ETag,
+		})
+
+		fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
+	}
+
+	if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		request := &filer_pb.CreateEntryRequest{
+			Directory: task.destinationUrlPath,
+			Entry: &filer_pb.Entry{
+				Name: fileName,
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:      time.Now().Unix(),
+					Mtime:       time.Now().Unix(),
+					Gid:         task.gid,
+					Uid:         task.uid,
+					FileSize:    uint64(task.fileSize),
+					FileMode:    uint32(task.fileMode),
+					Mime:        mimeType,
+					Replication: *worker.options.replication,
+					Collection:  *worker.options.collection,
+					TtlSec:      int32(util.ParseInt(*worker.options.ttl, 0)),
+				},
+				Chunks: chunks,
+			},
+		}
+
+		if _, err := client.CreateEntry(ctx, request); err != nil {
+			return fmt.Errorf("update fh: %v", err)
+		}
+		return nil
+	}); err != nil {
+		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
+	}
+
+	return nil
+}
+
+func (worker *FileCopyWorker) uploadFileInChunks(ctx context.Context, task FileCopyTask, f *os.File, chunkCount int, chunkSize int64) error {
+
+	fileName := filepath.Base(f.Name())
+	mimeType := detectMimeType(f)
+
+	var chunks []*filer_pb.FileChunk
+
+	for i := int64(0); i < int64(chunkCount); i++ {
+
+		// assign a volume
+		assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
+			Count:       1,
+			Replication: *worker.options.replication,
+			Collection:  *worker.options.collection,
+			Ttl:         *worker.options.ttl,
+		})
+		if err != nil {
+			fmt.Printf("Failed to assign from %s: %v\n", *worker.options.master, err)
+		}
+
+		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
+
+		uploadResult, err := operation.Upload(targetUrl,
+			fileName+"-"+strconv.FormatInt(i+1, 10),
+			io.LimitReader(f, chunkSize),
+			false, "application/octet-stream", nil, assignResult.Auth)
+		if err != nil {
+			return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
+		}
+		if uploadResult.Error != "" {
+			return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
+		}
+		chunks = append(chunks, &filer_pb.FileChunk{
+			FileId: assignResult.Fid,
+			Offset: i * chunkSize,
+			Size:   uint64(uploadResult.Size),
+			Mtime:  time.Now().UnixNano(),
+			ETag:   uploadResult.ETag,
+		})
+		fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
 	}
 
-	if err = filer_operation.RegisterFile(host, path, results[0].Fid, copy.secret); err != nil {
-		fmt.Printf("Failed to register file %s on %s: %v", fileOrDir, host, err)
-		return false
+	if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		request := &filer_pb.CreateEntryRequest{
+			Directory: task.destinationUrlPath,
+			Entry: &filer_pb.Entry{
+				Name: fileName,
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:      time.Now().Unix(),
+					Mtime:       time.Now().Unix(),
+					Gid:         task.gid,
+					Uid:         task.uid,
+					FileSize:    uint64(task.fileSize),
+					FileMode:    uint32(task.fileMode),
+					Mime:        mimeType,
+					Replication: *worker.options.replication,
+					Collection:  *worker.options.collection,
+					TtlSec:      int32(util.ParseInt(*worker.options.ttl, 0)),
+				},
+				Chunks: chunks,
+			},
+		}
+
+		if _, err := client.CreateEntry(ctx, request); err != nil {
+			return fmt.Errorf("update fh: %v", err)
+		}
+		return nil
+	}); err != nil {
+		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
 	}
+
+	fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
+
+	return nil
+}
+
+func detectMimeType(f *os.File) string {
+	head := make([]byte, 512)
+	f.Seek(0, io.SeekStart)
+	n, err := f.Read(head)
+	if err == io.EOF {
+		return ""
+	}
+	if err != nil {
+		fmt.Printf("read head of %v: %v\n", f.Name(), err)
+		return "application/octet-stream"
+	}
+	f.Seek(0, io.SeekStart)
+	mimeType := http.DetectContentType(head[:n])
+	return mimeType
+}
+
+func withFilerClient(ctx context.Context, filerAddress string, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	fmt.Printf("Copy %s => http://%s%s\n", fileOrDir, host, path)
+	return util.WithCachedGrpcClient(ctx, func(clientConn *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(clientConn)
+		return fn(client)
+	}, filerAddress, grpcDialOption)
 
-	return true
 }

weed/command/filer_replication.go

@@ -0,0 +1,141 @@
+package command
+
+import (
+	"context"
+	"strings"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/replication"
+	"github.com/chrislusf/seaweedfs/weed/replication/sink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/azuresink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/b2sink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/filersink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink"
+	"github.com/chrislusf/seaweedfs/weed/replication/sub"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
+)
+
+func init() {
+	cmdFilerReplicate.Run = runFilerReplicate // break init cycle
+}
+
+var cmdFilerReplicate = &Command{
+	UsageLine: "filer.replicate",
+	Short:     "replicate file changes to another destination",
+	Long: `replicate file changes to another destination
+
+	filer.replicate listens on filer notifications. If any file is updated, it will fetch the updated content,
+	and write to the other destination.
+
+	Run "weed scaffold -config=replication" to generate a replication.toml file and customize the parameters.
+
+  `,
+}
+
+func runFilerReplicate(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("security", false)
+	weed_server.LoadConfiguration("replication", true)
+	weed_server.LoadConfiguration("notification", true)
+	config := viper.GetViper()
+
+	var notificationInput sub.NotificationInput
+
+	validateOneEnabledInput(config)
+
+	for _, input := range sub.NotificationInputs {
+		if config.GetBool("notification." + input.GetName() + ".enabled") {
+			viperSub := config.Sub("notification." + input.GetName())
+			if err := input.Initialize(viperSub); err != nil {
+				glog.Fatalf("Failed to initialize notification input for %s: %+v",
+					input.GetName(), err)
+			}
+			glog.V(0).Infof("Configure notification input to %s", input.GetName())
+			notificationInput = input
+			break
+		}
+	}
+
+	if notificationInput == nil {
+		println("No notification is defined in notification.toml file.")
+		println("Please follow 'weed scaffold -config=notification' to see example notification configurations.")
+		return true
+	}
+
+	// avoid recursive replication
+	if config.GetBool("notification.source.filer.enabled") && config.GetBool("notification.sink.filer.enabled") {
+		sourceConfig, sinkConfig := config.Sub("source.filer"), config.Sub("sink.filer")
+		if sourceConfig.GetString("grpcAddress") == sinkConfig.GetString("grpcAddress") {
+			fromDir := sourceConfig.GetString("directory")
+			toDir := sinkConfig.GetString("directory")
+			if strings.HasPrefix(toDir, fromDir) {
+				glog.Fatalf("recursive replication! source directory %s includes the sink directory %s", fromDir, toDir)
+			}
+		}
+	}
+
+	var dataSink sink.ReplicationSink
+	for _, sk := range sink.Sinks {
+		if config.GetBool("sink." + sk.GetName() + ".enabled") {
+			viperSub := config.Sub("sink." + sk.GetName())
+			if err := sk.Initialize(viperSub); err != nil {
+				glog.Fatalf("Failed to initialize sink for %s: %+v",
+					sk.GetName(), err)
+			}
+			glog.V(0).Infof("Configure sink to %s", sk.GetName())
+			dataSink = sk
+			break
+		}
+	}
+
+	if dataSink == nil {
+		println("no data sink configured in replication.toml:")
+		for _, sk := range sink.Sinks {
+			println("    " + sk.GetName())
+		}
+		return true
+	}
+
+	replicator := replication.NewReplicator(config.Sub("source.filer"), dataSink)
+
+	for {
+		key, m, err := notificationInput.ReceiveMessage()
+		if err != nil {
+			glog.Errorf("receive %s: %+v", key, err)
+			continue
+		}
+		if key == "" {
+			// long poll received no messages
+			continue
+		}
+		if m.OldEntry != nil && m.NewEntry == nil {
+			glog.V(1).Infof("delete: %s", key)
+		} else if m.OldEntry == nil && m.NewEntry != nil {
+			glog.V(1).Infof("   add: %s", key)
+		} else {
+			glog.V(1).Infof("modify: %s", key)
+		}
+		if err = replicator.Replicate(context.Background(), key, m); err != nil {
+			glog.Errorf("replicate %s: %+v", key, err)
+		} else {
+			glog.V(1).Infof("replicated %s", key)
+		}
+	}
+
+	return true
+}
+
+func validateOneEnabledInput(config *viper.Viper) {
+	enabledInput := ""
+	for _, input := range sub.NotificationInputs {
+		if config.GetBool("notification." + input.GetName() + ".enabled") {
+			if enabledInput == "" {
+				enabledInput = input.GetName()
+			} else {
+				glog.Fatalf("Notification input is enabled for both %s and %s", enabledInput, input.GetName())
+			}
+		}
+	}
+}

weed/command/fix.go

@@ -7,6 +7,8 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 
 func init() {
@@ -27,43 +29,61 @@ var (
 	fixVolumeId         = cmdFix.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
 )
 
+type VolumeFileScanner4Fix struct {
+	version needle.Version
+	nm      *storage.NeedleMap
+}
+
+func (scanner *VolumeFileScanner4Fix) VisitSuperBlock(superBlock storage.SuperBlock) error {
+	scanner.version = superBlock.Version()
+	return nil
+
+}
+func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
+	return false
+}
+
+func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64) error {
+	glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped())
+	if n.Size > 0 && n.Size != types.TombstoneFileSize {
+		pe := scanner.nm.Put(n.Id, types.ToOffset(offset), n.Size)
+		glog.V(2).Infof("saved %d with error %v", n.Size, pe)
+	} else {
+		glog.V(2).Infof("skipping deleted file ...")
+		return scanner.nm.Delete(n.Id, types.ToOffset(offset))
+	}
+	return nil
+}
+
 func runFix(cmd *Command, args []string) bool {
 
 	if *fixVolumeId == -1 {
 		return false
 	}
 
-	fileName := strconv.Itoa(*fixVolumeId)
+	baseFileName := strconv.Itoa(*fixVolumeId)
 	if *fixVolumeCollection != "" {
-		fileName = *fixVolumeCollection + "_" + fileName
+		baseFileName = *fixVolumeCollection + "_" + baseFileName
 	}
-	indexFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	indexFileName := path.Join(*fixVolumePath, baseFileName+".idx")
+	indexFile, err := os.OpenFile(indexFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
 	if err != nil {
 		glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
 	}
 	defer indexFile.Close()
 
-	nm := storage.NewNeedleMap(indexFile)
+	nm := storage.NewBtreeNeedleMap(indexFile)
 	defer nm.Close()
 
-	vid := storage.VolumeId(*fixVolumeId)
-	err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid,
-		storage.NeedleMapInMemory,
-		func(superBlock storage.SuperBlock) error {
-			return nil
-		}, false, func(n *storage.Needle, offset int64) error {
-			glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped())
-			if n.Size > 0 {
-				pe := nm.Put(n.Id, uint32(offset/storage.NeedlePaddingSize), n.Size)
-				glog.V(2).Infof("saved %d with error %v", n.Size, pe)
-			} else {
-				glog.V(2).Infof("skipping deleted file ...")
-				return nm.Delete(n.Id, uint32(offset/storage.NeedlePaddingSize))
-			}
-			return nil
-		})
+	vid := needle.VolumeId(*fixVolumeId)
+	scanner := &VolumeFileScanner4Fix{
+		nm: nm,
+	}
+
+	err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid, storage.NeedleMapInMemory, scanner)
 	if err != nil {
 		glog.Fatalf("Export Volume File [ERROR] %s\n", err)
+		os.Remove(indexFileName)
 	}
 
 	return true

weed/command/master.go

@@ -1,22 +1,20 @@
 package command
 
 import (
-	"net/http"
-	"os"
-	"runtime"
-	"runtime/pprof"
-	"strconv"
-	"strings"
-	"time"
-
+	"github.com/chrislusf/raft/protobuf"
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
-	"github.com/soheilhy/cmux"
-	"google.golang.org/grpc"
+	"github.com/spf13/viper"
 	"google.golang.org/grpc/reflection"
+	"net/http"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
 )
 
 func init() {
@@ -26,8 +24,11 @@ func init() {
 var cmdMaster = &Command{
 	UsageLine: "master -port=9333",
 	Short:     "start a master server",
-	Long: `start a master server to provide volume=>location mapping service
-  and sequence number of file ids
+	Long: `start a master server to provide volume=>location mapping service and sequence number of file ids
+
+	The configuration file "security.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.
+
+	The example security.toml configuration file can be generated by "weed scaffold -config=security"
 
   `,
 }
@@ -37,91 +38,112 @@ var (
 	masterIp                = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address")
 	masterBindIp            = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
 	metaFolder              = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
-	masterPeers             = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094")
+	masterPeers             = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094")
 	volumeSizeLimitMB       = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
 	volumePreallocate       = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
 	mpulse                  = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
-	confFile                = cmdMaster.Flag.String("conf", "/etc/weedfs/weedfs.conf", "Deprecating! xml configuration file")
 	defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
 	// mTimeout                = cmdMaster.Flag.Int("idleTimeout", 30, "connection idle seconds")
 	mMaxCpu               = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
-	garbageThreshold      = cmdMaster.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
+	garbageThreshold      = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
-	masterSecureKey       = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+	disableHttp           = cmdMaster.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
 	masterCpuProfile      = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")
+	masterMemProfile      = cmdMaster.Flag.String("memprofile", "", "memory profile output file")
 
 	masterWhiteList []string
 )
 
 func runMaster(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("security", false)
+
 	if *mMaxCpu < 1 {
 		*mMaxCpu = runtime.NumCPU()
 	}
 	runtime.GOMAXPROCS(*mMaxCpu)
-	if *masterCpuProfile != "" {
-		f, err := os.Create(*masterCpuProfile)
-		if err != nil {
-			glog.Fatal(err)
-		}
-		pprof.StartCPUProfile(f)
-		defer pprof.StopCPUProfile()
-		OnInterrupt(func() {
-			pprof.StopCPUProfile()
-		})
-	}
+	util.SetupProfiling(*masterCpuProfile, *masterMemProfile)
+
 	if err := util.TestFolderWritable(*metaFolder); err != nil {
 		glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err)
 	}
 	if *masterWhiteListOption != "" {
 		masterWhiteList = strings.Split(*masterWhiteListOption, ",")
 	}
+	if *volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
+		glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
+	}
 
 	r := mux.NewRouter()
 	ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
 		*volumeSizeLimitMB, *volumePreallocate,
-		*mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold,
-		masterWhiteList, *masterSecureKey,
+		*mpulse, *defaultReplicaPlacement, *garbageThreshold,
+		masterWhiteList,
+		*disableHttp,
 	)
 
 	listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport)
 
 	glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)
 
-	listener, e := util.NewListener(listeningAddress, 0)
+	masterListener, e := util.NewListener(listeningAddress, 0)
 	if e != nil {
 		glog.Fatalf("Master startup error: %v", e)
 	}
 
 	go func() {
-		time.Sleep(100 * time.Millisecond)
-		myMasterAddress := *masterIp + ":" + strconv.Itoa(*mport)
-		var peers []string
-		if *masterPeers != "" {
-			peers = strings.Split(*masterPeers, ",")
-		}
-		raftServer := weed_server.NewRaftServer(r, peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
+		// start raftServer
+		myMasterAddress, peers := checkPeers(*masterIp, *mport, *masterPeers)
+		raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
+			peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
 		ms.SetRaftServer(raftServer)
+		r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
+
+		// starting grpc server
+		grpcPort := *mport + 10000
+		grpcL, err := util.NewListener(*masterBindIp+":"+strconv.Itoa(grpcPort), 0)
+		if err != nil {
+			glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
+		}
+		// Create your protocol servers.
+		grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
+		master_pb.RegisterSeaweedServer(grpcS, ms)
+		protobuf.RegisterRaftServer(grpcS, raftServer)
+		reflection.Register(grpcS)
+
+		glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *masterBindIp, grpcPort)
+		grpcS.Serve(grpcL)
 	}()
 
-	// start grpc and http server
-	m := cmux.New(listener)
-
-	grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
-	httpL := m.Match(cmux.Any())
-
-	// Create your protocol servers.
-	grpcS := grpc.NewServer()
-	pb.RegisterSeaweedServer(grpcS, ms)
-	reflection.Register(grpcS)
-
+	// start http server
 	httpS := &http.Server{Handler: r}
-
-	go grpcS.Serve(grpcL)
-	go httpS.Serve(httpL)
-
-	if err := m.Serve(); err != nil {
+	if err := httpS.Serve(masterListener); err != nil {
 		glog.Fatalf("master server failed to serve: %v", err)
 	}
 
 	return true
 }
+
+func checkPeers(masterIp string, masterPort int, peers string) (masterAddress string, cleanedPeers []string) {
+	masterAddress = masterIp + ":" + strconv.Itoa(masterPort)
+	if peers != "" {
+		cleanedPeers = strings.Split(peers, ",")
+	}
+
+	hasSelf := false
+	for _, peer := range cleanedPeers {
+		if peer == masterAddress {
+			hasSelf = true
+			break
+		}
+	}
+
+	peerCount := len(cleanedPeers)
+	if !hasSelf {
+		peerCount += 1
+	}
+	if peerCount%2 == 0 {
+		glog.Fatalf("Only odd number of masters are supported!")
+	}
+	return
+}

weed/command/mount.go

@@ -1,19 +1,44 @@
 package command
 
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
 type MountOptions struct {
-	filer *string
-	dir   *string
+	filer              *string
+	filerMountRootPath *string
+	dir                *string
+	dirListingLimit    *int
+	collection         *string
+	replication        *string
+	ttlSec             *int
+	chunkSizeLimitMB   *int
+	dataCenter         *string
+	allowOthers        *bool
 }
 
 var (
-	mountOptions MountOptions
+	mountOptions    MountOptions
+	mountCpuProfile *string
+	mountMemProfile *string
 )
 
 func init() {
 	cmdMount.Run = runMount // break init cycle
-	cmdMount.IsDebug = cmdMount.Flag.Bool("debug", false, "verbose debug information")
 	mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location")
+	mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server")
 	mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
+	mountOptions.dirListingLimit = cmdMount.Flag.Int("dirListLimit", 100000, "limit directory listing size")
+	mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files")
+	mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
+	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
+	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 4, "local write buffer size, also chunk large files")
+	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
+	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
+	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
+	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
 }
 
 var cmdMount = &Command{
@@ -26,10 +51,26 @@ var cmdMount = &Command{
   2) have a "weed filer" running
   These 2 requirements can be achieved with one command "weed server -filer=true"
 
-  This uses bazil.org/fuse, whichenables writing FUSE file systems on
+  This uses github.com/seaweedfs/fuse, which enables writing FUSE file systems on
   Linux, and OS X.
 
   On OS X, it requires OSXFUSE (http://osxfuse.github.com/).
 
   `,
 }
+
+func parseFilerGrpcAddress(filer string) (filerGrpcAddress string, err error) {
+	hostnameAndPort := strings.Split(filer, ":")
+	if len(hostnameAndPort) != 2 {
+		return "", fmt.Errorf("The filer should have hostname:port format: %v", hostnameAndPort)
+	}
+
+	filerPort, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64)
+	if parseErr != nil {
+		return "", fmt.Errorf("The filer filer port parse error: %v", parseErr)
+	}
+
+	filerGrpcPort := int(filerPort) + 10000
+
+	return fmt.Sprintf("%s:%d", hostnameAndPort[0], filerGrpcPort), nil
+}

weed/command/mount_std.go

@@ -4,36 +4,119 @@ package command
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
+	"os"
+	"os/user"
+	"path"
 	"runtime"
+	"strconv"
+	"strings"
+	"time"
 
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
-	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/filesys"
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"golang.org/x/net/context"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
 )
 
 func runMount(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("security", false)
+
 	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
 	if *mountOptions.dir == "" {
 		fmt.Printf("Please specify the mount directory via \"-dir\"")
 		return false
 	}
+	if *mountOptions.chunkSizeLimitMB <= 0 {
+		fmt.Printf("Please specify a reasonable buffer size.")
+		return false
+	}
 
-	c, err := fuse.Mount(*mountOptions.dir)
+	fuse.Unmount(*mountOptions.dir)
+
+	// detect mount folder mode
+	mountMode := os.ModeDir | 0755
+	if fileInfo, err := os.Stat(*mountOptions.dir); err == nil {
+		mountMode = os.ModeDir | fileInfo.Mode()
+	}
+
+	mountName := path.Base(*mountOptions.dir)
+
+	// detect current user
+	uid, gid := uint32(0), uint32(0)
+	if u, err := user.Current(); err == nil {
+		if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
+			uid = uint32(parsedId)
+		}
+		if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
+			gid = uint32(parsedId)
+		}
+	}
+
+	util.SetupProfiling(*mountCpuProfile, *mountMemProfile)
+
+	options := []fuse.MountOption{
+		fuse.VolumeName(mountName),
+		fuse.FSName("SeaweedFS"),
+		fuse.Subtype("SeaweedFS"),
+		fuse.NoAppleDouble(),
+		fuse.NoAppleXattr(),
+		fuse.NoBrowse(),
+		fuse.AutoXattr(),
+		fuse.ExclCreate(),
+		fuse.DaemonTimeout("3600"),
+		fuse.AllowSUID(),
+		fuse.DefaultPermissions(),
+		fuse.MaxReadahead(1024 * 128),
+		fuse.AsyncRead(),
+		fuse.WritebackCache(),
+		fuse.AllowNonEmptyMount(),
+	}
+	if *mountOptions.allowOthers {
+		options = append(options, fuse.AllowOther())
+	}
+
+	c, err := fuse.Mount(*mountOptions.dir, options...)
 	if err != nil {
 		glog.Fatal(err)
 		return false
 	}
 
-	OnInterrupt(func() {
+	util.OnInterrupt(func() {
 		fuse.Unmount(*mountOptions.dir)
 		c.Close()
 	})
 
-	err = fs.Serve(c, WFS{})
+	filerGrpcAddress, err := parseFilerGrpcAddress(*mountOptions.filer)
+	if err != nil {
+		glog.Fatal(err)
+		return false
+	}
+
+	mountRoot := *mountOptions.filerMountRootPath
+	if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") {
+		mountRoot = mountRoot[0 : len(mountRoot)-1]
+	}
+
+	err = fs.Serve(c, filesys.NewSeaweedFileSystem(&filesys.Option{
+		FilerGrpcAddress:   filerGrpcAddress,
+		GrpcDialOption:     security.LoadClientTLS(viper.Sub("grpc"), "client"),
+		FilerMountRootPath: mountRoot,
+		Collection:         *mountOptions.collection,
+		Replication:        *mountOptions.replication,
+		TtlSec:             int32(*mountOptions.ttlSec),
+		ChunkSizeLimit:     int64(*mountOptions.chunkSizeLimitMB) * 1024 * 1024,
+		DataCenter:         *mountOptions.dataCenter,
+		DirListingLimit:    *mountOptions.dirListingLimit,
+		EntryCacheTtl:      3 * time.Second,
+		MountUid:           uid,
+		MountGid:           gid,
+		MountMode:          mountMode,
+	}))
 	if err != nil {
 		fuse.Unmount(*mountOptions.dir)
 	}
@@ -46,61 +129,3 @@ func runMount(cmd *Command, args []string) bool {
 
 	return true
 }
-
-type File struct {
-	FileId filer.FileId
-	Name   string
-}
-
-func (File) Attr(context context.Context, attr *fuse.Attr) error {
-	return nil
-}
-func (File) ReadAll(ctx context.Context) ([]byte, error) {
-	return []byte("hello, world\n"), nil
-}
-
-type Dir struct {
-	Path string
-	Id   uint64
-}
-
-func (dir Dir) Attr(context context.Context, attr *fuse.Attr) error {
-	return nil
-}
-
-func (dir Dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
-	files_result, e := filer.ListFiles(*mountOptions.filer, dir.Path, name)
-	if e != nil {
-		return nil, fuse.ENOENT
-	}
-	if len(files_result.Files) > 0 {
-		return File{files_result.Files[0].Id, files_result.Files[0].Name}, nil
-	}
-	return nil, fmt.Errorf("File Not Found for %s", name)
-}
-
-type WFS struct{}
-
-func (WFS) Root() (fs.Node, error) {
-	return Dir{}, nil
-}
-
-func (dir *Dir) ReadDir(ctx context.Context) ([]fuse.Dirent, error) {
-	var ret []fuse.Dirent
-	if dirs, e := filer.ListDirectories(*mountOptions.filer, dir.Path); e == nil {
-		for _, d := range dirs.Directories {
-			dirId := uint64(d.Id)
-			ret = append(ret, fuse.Dirent{Inode: dirId, Name: d.Name, Type: fuse.DT_Dir})
-		}
-	}
-	if files, e := filer.ListFiles(*mountOptions.filer, dir.Path, ""); e == nil {
-		for _, f := range files.Files {
-			if fileId, e := storage.ParseFileId(string(f.Id)); e == nil {
-				fileInode := uint64(fileId.VolumeId)<<48 + fileId.Key
-				ret = append(ret, fuse.Dirent{Inode: fileInode, Name: f.Name, Type: fuse.DT_File})
-			}
-
-		}
-	}
-	return ret, nil
-}

weed/command/s3.go

@@ -0,0 +1,99 @@
+package command
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
+	"net/http"
+	"time"
+
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/s3api"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/gorilla/mux"
+)
+
+var (
+	s3StandaloneOptions S3Options
+)
+
+type S3Options struct {
+	filer            *string
+	filerBucketsPath *string
+	port             *int
+	domainName       *string
+	tlsPrivateKey    *string
+	tlsCertificate   *string
+}
+
+func init() {
+	cmdS3.Run = runS3 // break init cycle
+	s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
+	s3StandaloneOptions.filerBucketsPath = cmdS3.Flag.String("filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
+	s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
+	s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name, {bucket}.{domainName}")
+	s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
+	s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
+}
+
+var cmdS3 = &Command{
+	UsageLine: "s3 -port=8333 -filer=<ip:port>",
+	Short:     "start a s3 API compatible server that is backed by a filer",
+	Long: `start a s3 API compatible server that is backed by a filer.
+
+`,
+}
+
+func runS3(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("security", false)
+
+	return s3StandaloneOptions.startS3Server()
+
+}
+
+func (s3opt *S3Options) startS3Server() bool {
+
+	filerGrpcAddress, err := parseFilerGrpcAddress(*s3opt.filer)
+	if err != nil {
+		glog.Fatal(err)
+		return false
+	}
+
+	router := mux.NewRouter().SkipClean(true)
+
+	_, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
+		Filer:            *s3opt.filer,
+		FilerGrpcAddress: filerGrpcAddress,
+		DomainName:       *s3opt.domainName,
+		BucketsPath:      *s3opt.filerBucketsPath,
+		GrpcDialOption:   security.LoadClientTLS(viper.Sub("grpc"), "client"),
+	})
+	if s3ApiServer_err != nil {
+		glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
+	}
+
+	httpS := &http.Server{Handler: router}
+
+	listenAddress := fmt.Sprintf(":%d", *s3opt.port)
+	s3ApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
+	if err != nil {
+		glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
+	}
+
+	if *s3opt.tlsPrivateKey != "" {
+		glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.VERSION, *s3opt.port)
+		if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
+			glog.Fatalf("S3 API Server Fail to serve: %v", err)
+		}
+	} else {
+		glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.VERSION, *s3opt.port)
+		if err = httpS.Serve(s3ApiListener); err != nil {
+			glog.Fatalf("S3 API Server Fail to serve: %v", err)
+		}
+	}
+
+	return true
+
+}

weed/command/scaffold.go

@@ -0,0 +1,305 @@
+package command
+
+import (
+	"io/ioutil"
+	"path/filepath"
+)
+
+func init() {
+	cmdScaffold.Run = runScaffold // break init cycle
+}
+
+var cmdScaffold = &Command{
+	UsageLine: "scaffold -config=[filer|notification|replication|security]",
+	Short:     "generate basic configuration files",
+	Long: `Generate filer.toml with all possible configurations for you to customize.
+
+  `,
+}
+
+var (
+	outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
+	config     = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security] the configuration file to generate")
+)
+
+func runScaffold(cmd *Command, args []string) bool {
+
+	content := ""
+	switch *config {
+	case "filer":
+		content = FILER_TOML_EXAMPLE
+	case "notification":
+		content = NOTIFICATION_TOML_EXAMPLE
+	case "replication":
+		content = REPLICATION_TOML_EXAMPLE
+	case "security":
+		content = SECURITY_TOML_EXAMPLE
+	}
+	if content == "" {
+		println("need a valid -config option")
+		return false
+	}
+
+	if *outputPath != "" {
+		ioutil.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
+	} else {
+		println(content)
+	}
+	return true
+}
+
+const (
+	FILER_TOML_EXAMPLE = `
+# A sample TOML config file for SeaweedFS filer store
+# Used with "weed filer" or "weed server -filer"
+# Put this file to one of the location, with descending priority
+#    ./filer.toml
+#    $HOME/.seaweedfs/filer.toml
+#    /etc/seaweedfs/filer.toml
+
+[memory]
+# local in memory, mostly for testing purpose
+enabled = false
+
+[leveldb]
+# local on disk, mostly for simple single-machine setup, fairly scalable
+enabled = true
+dir = "."					# directory to store level db files
+
+####################################################
+# multiple filers on shared storage, fairly scalable
+####################################################
+
+[mysql]
+# CREATE TABLE IF NOT EXISTS filemeta (
+#   dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
+#   name        VARCHAR(1000)  COMMENT 'directory or file name',
+#   directory   TEXT           COMMENT 'full path to parent directory',
+#   meta        BLOB,
+#   PRIMARY KEY (dirhash, name)
+# ) DEFAULT CHARSET=utf8;
+
+enabled = false
+hostname = "localhost"
+port = 3306
+username = "root"
+password = ""
+database = ""              # create or use an existing database
+connection_max_idle = 2
+connection_max_open = 100
+
+[postgres]
+# CREATE TABLE IF NOT EXISTS filemeta (
+#   dirhash     BIGINT,
+#   name        VARCHAR(65535),
+#   directory   VARCHAR(65535),
+#   meta        bytea,
+#   PRIMARY KEY (dirhash, name)
+# );
+enabled = false
+hostname = "localhost"
+port = 5432
+username = "postgres"
+password = ""
+database = ""              # create or use an existing database
+sslmode = "disable"
+connection_max_idle = 100
+connection_max_open = 100
+
+[cassandra]
+# CREATE TABLE filemeta (
+#    directory varchar,
+#    name varchar,
+#    meta blob,
+#    PRIMARY KEY (directory, name)
+# ) WITH CLUSTERING ORDER BY (name ASC);
+enabled = false
+keyspace="seaweedfs"
+hosts=[
+	"localhost:9042",
+]
+
+[redis]
+enabled = false
+address  = "localhost:6379"
+password = ""
+db = 0
+
+[redis_cluster]
+enabled = false
+addresses = [
+    "localhost:30001",
+    "localhost:30002",
+    "localhost:30003",
+    "localhost:30004",
+    "localhost:30005",
+    "localhost:30006",
+]
+
+`
+
+	NOTIFICATION_TOML_EXAMPLE = `
+# A sample TOML config file for SeaweedFS filer store
+# Used by both "weed filer" or "weed server -filer" and "weed filer.replicate"
+# Put this file to one of the location, with descending priority
+#    ./notification.toml
+#    $HOME/.seaweedfs/notification.toml
+#    /etc/seaweedfs/notification.toml
+
+####################################################
+# notification
+# send and receive filer updates for each file to an external message queue
+####################################################
+[notification.log]
+# this is only for debugging perpose and does not work with "weed filer.replicate"
+enabled = false
+
+
+[notification.kafka]
+enabled = false
+hosts = [
+  "localhost:9092"
+]
+topic = "seaweedfs_filer"
+offsetFile = "./last.offset"
+offsetSaveIntervalSeconds = 10
+
+
+[notification.aws_sqs]
+# experimental, let me know if it works
+enabled = false
+aws_access_key_id     = ""        # if empty, loads from the shared credentials file (~/.aws/credentials).
+aws_secret_access_key = ""        # if empty, loads from the shared credentials file (~/.aws/credentials).
+region = "us-east-2"
+sqs_queue_name = "my_filer_queue" # an existing queue name
+
+
+[notification.google_pub_sub]
+# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
+enabled = false
+google_application_credentials = "/path/to/x.json" # path to json credential file
+project_id = ""                       # an existing project id
+topic = "seaweedfs_filer_topic"       # a topic, auto created if does not exists
+
+[notification.gocdk_pub_sub]
+# The Go Cloud Development Kit (https://gocloud.dev).
+# PubSub API (https://godoc.org/gocloud.dev/pubsub).
+# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
+enabled = false
+# This URL will Dial the RabbitMQ server at the URL in the environment
+# variable RABBIT_SERVER_URL and open the exchange "myexchange".
+# The exchange must have already been created by some other means, like
+# the RabbitMQ management plugin.
+topic_url = "rabbit://myexchange"
+sub_url = "rabbit://myqueue"
+`
+
+	REPLICATION_TOML_EXAMPLE = `
+# A sample TOML config file for replicating SeaweedFS filer
+# Used with "weed filer.replicate"
+# Put this file to one of the location, with descending priority
+#    ./replication.toml
+#    $HOME/.seaweedfs/replication.toml
+#    /etc/seaweedfs/replication.toml
+
+[source.filer]
+enabled = true
+grpcAddress = "localhost:18888"
+# all files under this directory tree are replicated.
+# this is not a directory on your hard drive, but on your filer.
+# i.e., all files with this "prefix" are sent to notification message queue.
+directory = "/buckets"    
+
+[sink.filer]
+enabled = false
+grpcAddress = "localhost:18888"
+# all replicated files are under this directory tree
+# this is not a directory on your hard drive, but on your filer.     
+# i.e., all received files will be "prefixed" to this directory.
+directory = "/backup"    
+replication = ""
+collection = ""
+ttlSec = 0
+
+[sink.s3]
+# read credentials doc at https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sessions.html
+# default loads credentials from the shared credentials file (~/.aws/credentials). 
+enabled = false
+aws_access_key_id     = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
+aws_secret_access_key = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
+region = "us-east-2"
+bucket = "your_bucket_name"    # an existing bucket
+directory = "/"                # destination directory
+
+[sink.google_cloud_storage]
+# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
+enabled = false
+google_application_credentials = "/path/to/x.json" # path to json credential file
+bucket = "your_bucket_seaweedfs"    # an existing bucket
+directory = "/"                     # destination directory
+
+[sink.azure]
+# experimental, let me know if it works
+enabled = false
+account_name = ""
+account_key  = ""
+container = "mycontainer"      # an existing container
+directory = "/"                # destination directory
+
+[sink.backblaze]
+enabled = false
+b2_account_id = ""
+b2_master_application_key  = ""
+bucket = "mybucket"            # an existing bucket
+directory = "/"                # destination directory
+
+`
+
+	SECURITY_TOML_EXAMPLE = `
+# Put this file to one of the location, with descending priority
+#    ./security.toml
+#    $HOME/.seaweedfs/security.toml
+#    /etc/seaweedfs/security.toml
+# this file is read by master, volume server, and filer
+
+# the jwt signing key is read by master and volume server
+# a jwt expires in 10 seconds
+[jwt.signing]
+key = ""
+
+# all grpc tls authentications are mutual
+# the values for the following ca, cert, and key are paths to the PERM files.
+[grpc]
+ca = ""
+
+[grpc.volume]
+cert = ""
+key  = ""
+
+[grpc.master]
+cert = ""
+key  = ""
+
+[grpc.filer]
+cert = ""
+key  = ""
+
+# use this for any place needs a grpc client
+# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
+[grpc.client]
+cert = ""
+key  = ""
+
+
+# volume server https options
+# Note: work in progress!
+#     this does not work with other clients, e.g., "weed filer|mount" etc, yet.
+[https.client]
+enabled = true
+[https.volume]
+cert = ""
+key  = ""
+
+
+`
+)

weed/command/server.go

@@ -1,6 +1,10 @@
 package command
 
 import (
+	"fmt"
+	"github.com/chrislusf/raft/protobuf"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
 	"net/http"
 	"os"
 	"runtime"
@@ -11,23 +15,22 @@ import (
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
-	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
-	"github.com/soheilhy/cmux"
-	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"
 )
 
 type ServerOptions struct {
 	cpuprofile *string
+	v          VolumeServerOptions
 }
 
 var (
 	serverOptions ServerOptions
 	filerOptions  FilerOptions
+	s3Options     S3Options
 )
 
 func init() {
@@ -36,17 +39,16 @@ func init() {
 
 var cmdServer = &Command{
 	UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name",
-	Short:     "start a server, including volume server, and automatically elect a master server",
+	Short:     "start a master server, a volume server, and optionally a filer and a S3 gateway",
 	Long: `start both a volume server to provide storage spaces
   and a master server to provide volume=>location mapping service and sequence number of file ids
 
   This is provided as a convenient way to start both volume server and master server.
-  The servers are exactly the same as starting them separately.
+  The servers acts exactly the same as starting them separately.
+  So other volume servers can connect to this master server also.
 
-  So other volume servers can use this embedded master server also.
-
-  Optionally, one filer server can be started. Logically, filer servers should not be in a cluster.
-  They run with meta data on disk, not shared. So each filer server is different.
+  Optionally, a filer server can be started.
+  Also optionally, a S3 gateway can be started.
 
   `,
 }
@@ -59,50 +61,54 @@ var (
 	serverDataCenter              = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
 	serverRack                    = cmdServer.Flag.String("rack", "", "current volume server's rack name")
 	serverWhiteListOption         = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
-	serverPeers                   = cmdServer.Flag.String("master.peers", "", "other master nodes in comma separated ip:masterPort list")
-	serverSecureKey               = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
-	serverGarbageThreshold        = cmdServer.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
+	serverDisableHttp             = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
+	serverPeers                   = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
+	serverGarbageThreshold        = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterPort                    = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
 	masterMetaFolder              = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
 	masterVolumeSizeLimitMB       = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
 	masterVolumePreallocate       = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
-	masterConfFile                = cmdServer.Flag.String("master.conf", "/etc/weedfs/weedfs.conf", "xml configuration file")
 	masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
-	volumePort                    = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
-	volumePublicPort              = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
 	volumeDataFolders             = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
 	volumeMaxDataVolumeCounts     = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
-	volumePulse                   = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
-	volumeIndexType               = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb] mode for memory~performance balance.")
-	volumeFixJpgOrientation       = cmdServer.Flag.Bool("volume.images.fix.orientation", true, "Adjust jpg orientation when uploading.")
-	volumeReadRedirect            = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
-	volumeServerPublicUrl         = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
-	volumeEnableBytesCache        = cmdServer.Flag.Bool("volume.cache.enable", false, "direct cache instead of OS cache, cost more memory.")
+	pulseSeconds                  = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
 	isStartingFiler               = cmdServer.Flag.Bool("filer", false, "whether to start filer")
+	isStartingS3                  = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
 
 	serverWhiteList []string
 )
 
 func init() {
 	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
-	filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server")
 	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
 	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
-	filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -dir is specified")
+	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
 	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
 	filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
 	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
-	filerOptions.confFile = cmdServer.Flag.String("filer.confFile", "", "json encoded filer conf file")
-	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 0, "split files larger than the limit")
-	filerOptions.cassandra_server = cmdServer.Flag.String("filer.cassandra.server", "", "host[:port] of the cassandra server")
-	filerOptions.cassandra_keyspace = cmdServer.Flag.String("filer.cassandra.keyspace", "seaweed", "keyspace of the cassandra server")
-	filerOptions.redis_server = cmdServer.Flag.String("filer.redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379")
-	filerOptions.redis_password = cmdServer.Flag.String("filer.redis.password", "", "redis password in clear text")
-	filerOptions.redis_database = cmdServer.Flag.Int("filer.redis.database", 0, "the database on the redis server")
+	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
+	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
+
+	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
+	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
+	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
+	serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
+	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
+	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
+	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
+
+	s3Options.filerBucketsPath = cmdServer.Flag.String("s3.filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
+	s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
+	s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}")
+	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
+	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
+
 }
 
 func runServer(cmd *Command, args []string) bool {
-	filerOptions.secretKey = serverSecureKey
+
+	weed_server.LoadConfiguration("security", false)
+
 	if *serverOptions.cpuprofile != "" {
 		f, err := os.Create(*serverOptions.cpuprofile)
 		if err != nil {
@@ -116,55 +122,50 @@ func runServer(cmd *Command, args []string) bool {
 		*isStartingFiler = true
 	}
 
-	*filerOptions.master = *serverIp + ":" + strconv.Itoa(*masterPort)
+	if *isStartingS3 {
+		*isStartingFiler = true
+	}
+
+	master := *serverIp + ":" + strconv.Itoa(*masterPort)
+	filerOptions.masters = &master
+	filerOptions.ip = serverIp
+	serverOptions.v.ip = serverIp
+	serverOptions.v.bindIp = serverBindIp
+	serverOptions.v.masters = &master
+	serverOptions.v.idleConnectionTimeout = serverTimeout
+	serverOptions.v.maxCpu = serverMaxCpu
+	serverOptions.v.dataCenter = serverDataCenter
+	serverOptions.v.rack = serverRack
+	serverOptions.v.pulseSeconds = pulseSeconds
+
+	filerOptions.dataCenter = serverDataCenter
+	filerOptions.disableHttp = serverDisableHttp
+
+	filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port)
+	s3Options.filer = &filerAddress
 
 	if *filerOptions.defaultReplicaPlacement == "" {
 		*filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement
 	}
 
-	if *volumePublicPort == 0 {
-		*volumePublicPort = *volumePort
-	}
-
 	if *serverMaxCpu < 1 {
 		*serverMaxCpu = runtime.NumCPU()
 	}
 	runtime.GOMAXPROCS(*serverMaxCpu)
 
 	folders := strings.Split(*volumeDataFolders, ",")
-	maxCountStrings := strings.Split(*volumeMaxDataVolumeCounts, ",")
-	var maxCounts []int
-	for _, maxString := range maxCountStrings {
-		if max, e := strconv.Atoi(maxString); e == nil {
-			maxCounts = append(maxCounts, max)
-		} else {
-			glog.Fatalf("The max specified in -max not a valid number %s", maxString)
-		}
-	}
-	if len(folders) != len(maxCounts) {
-		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
-	}
-	for _, folder := range folders {
-		if err := util.TestFolderWritable(folder); err != nil {
-			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
-		}
+
+	if *masterVolumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
+		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
 	}
 
 	if *masterMetaFolder == "" {
 		*masterMetaFolder = folders[0]
 	}
-	if *isStartingFiler {
-		if *filerOptions.dir == "" {
-			*filerOptions.dir = *masterMetaFolder + "/filer"
-			os.MkdirAll(*filerOptions.dir, 0700)
-		}
-		if err := util.TestFolderWritable(*filerOptions.dir); err != nil {
-			glog.Fatalf("Check Mapping Meta Folder (-filer.dir=\"%s\") Writable: %s", *filerOptions.dir, err)
-		}
-	}
 	if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
 		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
 	}
+	filerOptions.defaultLevelDbDirectory = masterMetaFolder
 
 	if *serverWhiteListOption != "" {
 		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
@@ -173,85 +174,69 @@ func runServer(cmd *Command, args []string) bool {
 	if *isStartingFiler {
 		go func() {
 			time.Sleep(1 * time.Second)
-			r := http.NewServeMux()
-			_, nfs_err := weed_server.NewFilerServer(r, *serverBindIp, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection,
-				*filerOptions.defaultReplicaPlacement,
-				*filerOptions.redirectOnRead, *filerOptions.disableDirListing,
-				*filerOptions.confFile,
-				*filerOptions.maxMB,
-				*filerOptions.secretKey,
-				*filerOptions.cassandra_server, *filerOptions.cassandra_keyspace,
-				*filerOptions.redis_server, *filerOptions.redis_password, *filerOptions.redis_database,
-			)
-			if nfs_err != nil {
-				glog.Fatalf("Filer startup error: %v", nfs_err)
-			}
-			glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*filerOptions.port))
-			filerListener, e := util.NewListener(
-				":"+strconv.Itoa(*filerOptions.port),
-				time.Duration(10)*time.Second,
-			)
-			if e != nil {
-				glog.Fatalf("Filer listener error: %v", e)
-			}
-			if e := http.Serve(filerListener, r); e != nil {
-				glog.Fatalf("Filer Fail to serve: %v", e)
-			}
+
+			filerOptions.startFiler()
+
+		}()
+	}
+
+	if *isStartingS3 {
+		go func() {
+			time.Sleep(2 * time.Second)
+
+			s3Options.startS3Server()
+
 		}()
 	}
 
-	var raftWaitForMaster sync.WaitGroup
 	var volumeWait sync.WaitGroup
 
-	raftWaitForMaster.Add(1)
 	volumeWait.Add(1)
 
 	go func() {
 		r := mux.NewRouter()
 		ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
 			*masterVolumeSizeLimitMB, *masterVolumePreallocate,
-			*volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
-			serverWhiteList, *serverSecureKey,
+			*pulseSeconds, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
+			serverWhiteList, *serverDisableHttp,
 		)
 
-		glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort))
+		glog.V(0).Infof("Start Seaweed Master %s at %s:%d", util.VERSION, *serverIp, *masterPort)
 		masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), 0)
 		if e != nil {
 			glog.Fatalf("Master startup error: %v", e)
 		}
 
 		go func() {
-			raftWaitForMaster.Wait()
-			time.Sleep(100 * time.Millisecond)
-			myAddress := *serverIp + ":" + strconv.Itoa(*masterPort)
-			var peers []string
-			if *serverPeers != "" {
-				peers = strings.Split(*serverPeers, ",")
-			}
-			raftServer := weed_server.NewRaftServer(r, peers, myAddress, *masterMetaFolder, ms.Topo, *volumePulse)
+			// start raftServer
+			myMasterAddress, peers := checkPeers(*serverIp, *masterPort, *serverPeers)
+			raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
+				peers, myMasterAddress, *masterMetaFolder, ms.Topo, *pulseSeconds)
 			ms.SetRaftServer(raftServer)
-			volumeWait.Done()
+			r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
+
+			// starting grpc server
+			grpcPort := *masterPort + 10000
+			grpcL, err := util.NewListener(*serverIp+":"+strconv.Itoa(grpcPort), 0)
+			if err != nil {
+				glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
+			}
+			// Create your protocol servers.
+			glog.V(0).Infof("grpc config %+v", viper.Sub("grpc"))
+			grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
+			master_pb.RegisterSeaweedServer(grpcS, ms)
+			protobuf.RegisterRaftServer(grpcS, raftServer)
+			reflection.Register(grpcS)
+
+			glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *serverIp, grpcPort)
+			grpcS.Serve(grpcL)
 		}()
 
-		raftWaitForMaster.Done()
-
-		// start grpc and http server
-		m := cmux.New(masterListener)
-
-		grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
-		httpL := m.Match(cmux.Any())
-
-		// Create your protocol servers.
-		grpcS := grpc.NewServer()
-		pb.RegisterSeaweedServer(grpcS, ms)
-		reflection.Register(grpcS)
+		volumeWait.Done()
 
+		// start http server
 		httpS := &http.Server{Handler: r}
-
-		go grpcS.Serve(grpcL)
-		go httpS.Serve(httpL)
-
-		if err := m.Serve(); err != nil {
+		if err := httpS.Serve(masterListener); err != nil {
 			glog.Fatalf("master server failed to serve: %v", err)
 		}
 
@@ -259,64 +244,8 @@ func runServer(cmd *Command, args []string) bool {
 
 	volumeWait.Wait()
 	time.Sleep(100 * time.Millisecond)
-	if *volumePublicPort == 0 {
-		*volumePublicPort = *volumePort
-	}
-	if *volumeServerPublicUrl == "" {
-		*volumeServerPublicUrl = *serverIp + ":" + strconv.Itoa(*volumePublicPort)
-	}
-	isSeperatedPublicPort := *volumePublicPort != *volumePort
-	volumeMux := http.NewServeMux()
-	publicVolumeMux := volumeMux
-	if isSeperatedPublicPort {
-		publicVolumeMux = http.NewServeMux()
-	}
-	volumeNeedleMapKind := storage.NeedleMapInMemory
-	switch *volumeIndexType {
-	case "leveldb":
-		volumeNeedleMapKind = storage.NeedleMapLevelDb
-	case "boltdb":
-		volumeNeedleMapKind = storage.NeedleMapBoltDb
-	}
-	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
-		*serverIp, *volumePort, *volumeServerPublicUrl,
-		folders, maxCounts,
-		volumeNeedleMapKind,
-		*serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack,
-		serverWhiteList, *volumeFixJpgOrientation, *volumeReadRedirect,
-		*volumeEnableBytesCache,
-	)
 
-	glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort))
-	volumeListener, eListen := util.NewListener(
-		*serverBindIp+":"+strconv.Itoa(*volumePort),
-		time.Duration(*serverTimeout)*time.Second,
-	)
-	if eListen != nil {
-		glog.Fatalf("Volume server listener error: %v", eListen)
-	}
-	if isSeperatedPublicPort {
-		publicListeningAddress := *serverIp + ":" + strconv.Itoa(*volumePublicPort)
-		glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "public at", publicListeningAddress)
-		publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*serverTimeout)*time.Second)
-		if e != nil {
-			glog.Fatalf("Volume server listener error:%v", e)
-		}
-		go func() {
-			if e := http.Serve(publicListener, publicVolumeMux); e != nil {
-				glog.Fatalf("Volume server fail to serve public: %v", e)
-			}
-		}()
-	}
-
-	OnInterrupt(func() {
-		volumeServer.Shutdown()
-		pprof.StopCPUProfile()
-	})
-
-	if e := http.Serve(volumeListener, volumeMux); e != nil {
-		glog.Fatalf("Volume server fail to serve:%v", e)
-	}
+	serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption)
 
 	return true
 }

weed/command/shell.go

@@ -1,21 +1,25 @@
 package command
 
 import (
-	"bufio"
-	"fmt"
-	"os"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/shell"
+	"github.com/spf13/viper"
+)
 
-	"github.com/chrislusf/seaweedfs/weed/glog"
+var (
+	shellOptions shell.ShellOptions
 )
 
 func init() {
 	cmdShell.Run = runShell // break init cycle
+	shellOptions.Masters = cmdShell.Flag.String("master", "localhost:9333", "comma-separated master servers")
 }
 
 var cmdShell = &Command{
 	UsageLine: "shell",
-	Short:     "run interactive commands, now just echo",
-	Long: `run interactive commands.
+	Short:     "run interactive administrative commands",
+	Long: `run interactive administrative commands.
 
   `,
 }
@@ -23,39 +27,16 @@ var cmdShell = &Command{
 var ()
 
 func runShell(command *Command, args []string) bool {
-	r := bufio.NewReader(os.Stdin)
-	o := bufio.NewWriter(os.Stdout)
-	e := bufio.NewWriter(os.Stderr)
-	prompt := func() {
-		var err error
-		if _, err = o.WriteString("> "); err != nil {
-			glog.V(0).Infoln("error writing to stdout:", err)
-		}
-		if err = o.Flush(); err != nil {
-			glog.V(0).Infoln("error flushing stdout:", err)
-		}
-	}
-	readLine := func() string {
-		ret, err := r.ReadString('\n')
-		if err != nil {
-			fmt.Fprint(e, err)
-			os.Exit(1)
-		}
-		return ret
-	}
-	execCmd := func(cmd string) int {
-		if cmd != "" {
-			if _, err := o.WriteString(cmd); err != nil {
-				glog.V(0).Infoln("error writing to stdout:", err)
-			}
-		}
-		return 0
-	}
 
-	cmd := ""
-	for {
-		prompt()
-		cmd = readLine()
-		execCmd(cmd)
-	}
+	weed_server.LoadConfiguration("security", false)
+	shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
+
+	shellOptions.FilerHost = "localhost"
+	shellOptions.FilerPort = 8888
+	shellOptions.Directory = "/"
+
+	shell.RunShell(shellOptions)
+
+	return true
+
 }

weed/command/upload.go

@@ -3,11 +3,13 @@ package command
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
 	"os"
 	"path/filepath"
 
 	"github.com/chrislusf/seaweedfs/weed/operation"
-	"github.com/chrislusf/seaweedfs/weed/security"
 )
 
 var (
@@ -23,7 +25,6 @@ type UploadOptions struct {
 	dataCenter  *string
 	ttl         *string
 	maxMB       *int
-	secretKey   *string
 }
 
 func init() {
@@ -36,12 +37,11 @@ func init() {
 	upload.collection = cmdUpload.Flag.String("collection", "", "optional collection name")
 	upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
 	upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
-	upload.maxMB = cmdUpload.Flag.Int("maxMB", 0, "split files larger than the limit")
-	upload.secretKey = cmdUpload.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+	upload.maxMB = cmdUpload.Flag.Int("maxMB", 32, "split files larger than the limit")
 }
 
 var cmdUpload = &Command{
-	UsageLine: "upload -master=localhost:9333 file1 [file2 file3]\n         weed upload -server=localhost:9333 -dir=one_directory -include=*.pdf",
+	UsageLine: "upload -master=localhost:9333 file1 [file2 file3]\n         weed upload -master=localhost:9333 -dir=one_directory -include=*.pdf",
 	Short:     "upload one or a list of files",
 	Long: `upload one or a list of files, or batch upload one whole folder recursively.
 
@@ -53,18 +53,17 @@ var cmdUpload = &Command{
   All files under the folder and subfolders will be uploaded, each with its own file key.
   Optional parameter "-include" allows you to specify the file name patterns.
 
-  If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
-  This can save volume server's gzipped processing and allow customizable gzip compression level.
-  The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
-
-  If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separatedly.
+  If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separately.
   The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
 
   `,
 }
 
 func runUpload(cmd *Command, args []string) bool {
-	secret := security.Secret(*upload.secretKey)
+
+	weed_server.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
 	if len(args) == 0 {
 		if *upload.dir == "" {
 			return false
@@ -81,9 +80,9 @@ func runUpload(cmd *Command, args []string) bool {
 					if e != nil {
 						return e
 					}
-					results, e := operation.SubmitFiles(*upload.master, parts,
+					results, e := operation.SubmitFiles(*upload.master, grpcDialOption, parts,
 						*upload.replication, *upload.collection, *upload.dataCenter,
-						*upload.ttl, *upload.maxMB, secret)
+						*upload.ttl, *upload.maxMB)
 					bytes, _ := json.Marshal(results)
 					fmt.Println(string(bytes))
 					if e != nil {
@@ -100,9 +99,9 @@ func runUpload(cmd *Command, args []string) bool {
 		if e != nil {
 			fmt.Println(e.Error())
 		}
-		results, _ := operation.SubmitFiles(*upload.master, parts,
+		results, _ := operation.SubmitFiles(*upload.master, grpcDialOption, parts,
 			*upload.replication, *upload.collection, *upload.dataCenter,
-			*upload.ttl, *upload.maxMB, secret)
+			*upload.ttl, *upload.maxMB)
 		bytes, _ := json.Marshal(results)
 		fmt.Println(string(bytes))
 	}

weed/command/volume.go

@@ -1,17 +1,22 @@
 package command
 
 import (
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
 	"net/http"
 	"os"
 	"runtime"
+	"runtime/pprof"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc/reflection"
 )
 
 var (
@@ -26,7 +31,7 @@ type VolumeServerOptions struct {
 	ip                    *string
 	publicUrl             *string
 	bindIp                *string
-	master                *string
+	masters               *string
 	pulseSeconds          *int
 	idleConnectionTimeout *int
 	maxCpu                *int
@@ -36,7 +41,9 @@ type VolumeServerOptions struct {
 	indexType             *string
 	fixJpgOrientation     *bool
 	readRedirect          *bool
-	enableBytesCache      *bool
+	cpuProfile            *string
+	memProfile            *string
+	compactionMBPerSecond *int
 }
 
 func init() {
@@ -46,16 +53,18 @@ func init() {
 	v.ip = cmdVolume.Flag.String("ip", "", "ip or server name")
 	v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
 	v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
-	v.master = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location")
+	v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
 	v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
 	v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
 	v.maxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
 	v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
 	v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
-	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|boltdb] mode for memory~performance balance.")
-	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", true, "Adjust jpg orientation when uploading.")
+	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
+	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
 	v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
-	v.enableBytesCache = cmdVolume.Flag.Bool("cache.enable", false, "direct cache instead of OS cache, cost more memory.")
+	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
+	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
+	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit compaction speed in mega bytes per second")
 }
 
 var cmdVolume = &Command{
@@ -73,14 +82,25 @@ var (
 )
 
 func runVolume(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("security", false)
+
 	if *v.maxCpu < 1 {
 		*v.maxCpu = runtime.NumCPU()
 	}
 	runtime.GOMAXPROCS(*v.maxCpu)
+	util.SetupProfiling(*v.cpuProfile, *v.memProfile)
+
+	v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption)
+
+	return true
+}
+
+func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption string) {
 
 	//Set multiple folders and each folder's max volume count limit'
-	v.folders = strings.Split(*volumeFolders, ",")
-	maxCountStrings := strings.Split(*maxVolumeCounts, ",")
+	v.folders = strings.Split(volumeFolders, ",")
+	maxCountStrings := strings.Split(maxVolumeCounts, ",")
 	for _, maxString := range maxCountStrings {
 		if max, e := strconv.Atoi(maxString); e == nil {
 			v.folderMaxLimits = append(v.folderMaxLimits, max)
@@ -98,8 +118,8 @@ func runVolume(cmd *Command, args []string) bool {
 	}
 
 	//security related white list configuration
-	if *volumeWhiteListOption != "" {
-		v.whiteList = strings.Split(*volumeWhiteListOption, ",")
+	if volumeWhiteListOption != "" {
+		v.whiteList = strings.Split(volumeWhiteListOption, ",")
 	}
 
 	if *v.ip == "" {
@@ -124,21 +144,26 @@ func runVolume(cmd *Command, args []string) bool {
 	switch *v.indexType {
 	case "leveldb":
 		volumeNeedleMapKind = storage.NeedleMapLevelDb
-	case "boltdb":
-		volumeNeedleMapKind = storage.NeedleMapBoltDb
+	case "leveldbMedium":
+		volumeNeedleMapKind = storage.NeedleMapLevelDbMedium
+	case "leveldbLarge":
+		volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
 	}
+
+	masters := *v.masters
+
 	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
 		*v.ip, *v.port, *v.publicUrl,
 		v.folders, v.folderMaxLimits,
 		volumeNeedleMapKind,
-		*v.master, *v.pulseSeconds, *v.dataCenter, *v.rack,
+		strings.Split(masters, ","), *v.pulseSeconds, *v.dataCenter, *v.rack,
 		v.whiteList,
 		*v.fixJpgOrientation, *v.readRedirect,
-		*v.enableBytesCache,
+		*v.compactionMBPerSecond,
 	)
 
 	listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
-	glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", listeningAddress)
+	glog.V(0).Infof("Start Seaweed volume server %s at %s", util.VERSION, listeningAddress)
 	listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
 	if e != nil {
 		glog.Fatalf("Volume server listener error:%v", e)
@@ -157,12 +182,31 @@ func runVolume(cmd *Command, args []string) bool {
 		}()
 	}
 
-	OnInterrupt(func() {
+	util.OnInterrupt(func() {
 		volumeServer.Shutdown()
+		pprof.StopCPUProfile()
 	})
 
-	if e := http.Serve(listener, volumeMux); e != nil {
-		glog.Fatalf("Volume server fail to serve: %v", e)
+	// starting grpc server
+	grpcPort := *v.port + 10000
+	grpcL, err := util.NewListener(*v.bindIp+":"+strconv.Itoa(grpcPort), 0)
+	if err != nil {
+		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
 	}
-	return true
+	grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "volume"))
+	volume_server_pb.RegisterVolumeServerServer(grpcS, volumeServer)
+	reflection.Register(grpcS)
+	go grpcS.Serve(grpcL)
+
+	if viper.GetString("https.volume.key") != "" {
+		if e := http.ServeTLS(listener, volumeMux,
+			viper.GetString("https.volume.cert"), viper.GetString("https.volume.key")); e != nil {
+			glog.Fatalf("Volume server fail to serve: %v", e)
+		}
+	} else {
+		if e := http.Serve(listener, volumeMux); e != nil {
+			glog.Fatalf("Volume server fail to serve: %v", e)
+		}
+	}
+
 }

weed/command/webdav.go

@@ -0,0 +1,109 @@
+package command
+
+import (
+	"fmt"
+	"net/http"
+	"os/user"
+	"strconv"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/spf13/viper"
+)
+
+var (
+	webDavStandaloneOptions WebDavOption
+)
+
+type WebDavOption struct {
+	filer          *string
+	port           *int
+	collection     *string
+	tlsPrivateKey  *string
+	tlsCertificate *string
+}
+
+func init() {
+	cmdWebDav.Run = runWebDav // break init cycle
+	webDavStandaloneOptions.filer = cmdWebDav.Flag.String("filer", "localhost:8888", "filer server address")
+	webDavStandaloneOptions.port = cmdWebDav.Flag.Int("port", 7333, "webdav server http listen port")
+	webDavStandaloneOptions.collection = cmdWebDav.Flag.String("collection", "", "collection to create the files")
+	webDavStandaloneOptions.tlsPrivateKey = cmdWebDav.Flag.String("key.file", "", "path to the TLS private key file")
+	webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file")
+}
+
+var cmdWebDav = &Command{
+	UsageLine: "webdav -port=7333 -filer=<ip:port>",
+	Short:     "start a webdav server that is backed by a filer",
+	Long: `start a webdav server that is backed by a filer.
+
+`,
+}
+
+func runWebDav(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("security", false)
+
+	glog.V(0).Infof("Starting Seaweed WebDav Server %s at https port %d", util.VERSION, *webDavStandaloneOptions.port)
+
+	return webDavStandaloneOptions.startWebDav()
+
+}
+
+func (wo *WebDavOption) startWebDav() bool {
+
+	filerGrpcAddress, err := parseFilerGrpcAddress(*wo.filer)
+	if err != nil {
+		glog.Fatal(err)
+		return false
+	}
+
+	// detect current user
+	uid, gid := uint32(0), uint32(0)
+	if u, err := user.Current(); err == nil {
+		if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
+			uid = uint32(parsedId)
+		}
+		if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
+			gid = uint32(parsedId)
+		}
+	}
+
+	ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{
+		Filer:            *wo.filer,
+		FilerGrpcAddress: filerGrpcAddress,
+		GrpcDialOption:   security.LoadClientTLS(viper.Sub("grpc"), "client"),
+		Collection:       *wo.collection,
+		Uid:              uid,
+		Gid:              gid,
+	})
+	if webdavServer_err != nil {
+		glog.Fatalf("WebDav Server startup error: %v", webdavServer_err)
+	}
+
+	httpS := &http.Server{Handler: ws.Handler}
+
+	listenAddress := fmt.Sprintf(":%d", *wo.port)
+	webDavListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
+	if err != nil {
+		glog.Fatalf("WebDav Server listener on %s error: %v", listenAddress, err)
+	}
+
+	if *wo.tlsPrivateKey != "" {
+		glog.V(0).Infof("Start Seaweed WebDav Server %s at https port %d", util.VERSION, *wo.port)
+		if err = httpS.ServeTLS(webDavListener, *wo.tlsCertificate, *wo.tlsPrivateKey); err != nil {
+			glog.Fatalf("WebDav Server Fail to serve: %v", err)
+		}
+	} else {
+		glog.V(0).Infof("Start Seaweed WebDav Server %s at http port %d", util.VERSION, *wo.port)
+		if err = httpS.Serve(webDavListener); err != nil {
+			glog.Fatalf("WebDav Server Fail to serve: %v", err)
+		}
+	}
+
+	return true
+
+}

weed/compress/compression_test.go

@@ -1,45 +0,0 @@
-package compress
-
-import (
-	"math/rand"
-	"testing"
-)
-
-func TestSortedData(t *testing.T) {
-	data := make([]int32, 102400)
-	for i := 1; i < len(data); i++ {
-		data[i] = data[i-1] + rand.Int31n(15)
-	}
-	testCompressAndUncompress(t, data, "Sorted data")
-}
-
-func TestUnsortedData(t *testing.T) {
-	data := make([]int32, 102400)
-	for i := 0; i < len(data); i++ {
-		data[i] = rand.Int31n(255)
-	}
-	testCompressAndUncompress(t, data, "Unsorted data")
-}
-
-func testCompressAndUncompress(t *testing.T, data []int32, desc string) {
-
-	compressed_data, err := Compress32(data)
-	if err != nil {
-		t.Fatal("Compress error", err.Error())
-	}
-	uncompressed_data, err := Uncompress32(compressed_data, make([]int32, len(data)*2))
-	if err != nil {
-		t.Fatal("Compress error", err.Error())
-	}
-	if len(uncompressed_data) != len(data) {
-		t.Fatal("Len differs", len(data), len(uncompressed_data))
-	}
-	for i := 0; i < len(data); i++ {
-		if data[i] != uncompressed_data[i] {
-			t.Fatal("Data differs:", i, data[i], uncompressed_data[i])
-		}
-	}
-
-	println(desc, " Data length:", len(data), " => Compressed length:", len(compressed_data))
-
-}

weed/compress/delta_binary_pack32.go

@@ -1,32 +0,0 @@
-package compress
-
-import (
-	"github.com/reducedb/encoding/cursor"
-	"github.com/reducedb/encoding/delta/bp32"
-)
-
-// Compress compresses in[]int32 to out[]int32
-func Compress32(in []int32) (out []int32, err error) {
-	out = make([]int32, len(in)*2)
-	inpos := cursor.New()
-	outpos := cursor.New()
-
-	if err = bp32.New().Compress(in, inpos, len(in), out, outpos); err != nil {
-		return nil, err
-	}
-
-	return out[:outpos.Get()], nil
-}
-
-// Uncompress uncompresses in[]int32 to out[]int32
-func Uncompress32(in []int32, buffer []int32) (out []int32, err error) {
-	out = buffer
-	inpos := cursor.New()
-	outpos := cursor.New()
-
-	if err = bp32.New().Uncompress(in, inpos, len(in), out, outpos); err != nil {
-		return nil, err
-	}
-
-	return out[:outpos.Get()], nil
-}

weed/filer/cassandra_store/cassandra_store.go

@@ -1,96 +0,0 @@
-package cassandra_store
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/chrislusf/seaweedfs/weed/filer"
-	"github.com/chrislusf/seaweedfs/weed/glog"
-
-	"github.com/gocql/gocql"
-)
-
-/*
-
-Basically you need a table just like this:
-
-CREATE TABLE seaweed_files (
-   path varchar,
-   fids list<varchar>,
-   PRIMARY KEY (path)
-);
-
-Need to match flat_namespace.FlatNamespaceStore interface
-	Put(fullFileName string, fid string) (err error)
-	Get(fullFileName string) (fid string, err error)
-	Delete(fullFileName string) (fid string, err error)
-
-*/
-type CassandraStore struct {
-	cluster *gocql.ClusterConfig
-	session *gocql.Session
-}
-
-func NewCassandraStore(keyspace string, hosts string) (c *CassandraStore, err error) {
-	c = &CassandraStore{}
-	s := strings.Split(hosts, ",")
-	if len(s) == 1 {
-		glog.V(2).Info("Only one cassandra node to connect! A cluster is Recommended! Now using:", string(hosts))
-		c.cluster = gocql.NewCluster(hosts)
-	} else if len(s) > 1 {
-		c.cluster = gocql.NewCluster(s...)
-	}
-	c.cluster.Keyspace = keyspace
-	c.cluster.Consistency = gocql.Quorum
-	c.session, err = c.cluster.CreateSession()
-	if err != nil {
-		glog.V(0).Infof("Failed to open cassandra store, hosts %v, keyspace %s", hosts, keyspace)
-	}
-	return
-}
-
-func (c *CassandraStore) Put(fullFileName string, fid string) (err error) {
-	var input []string
-	input = append(input, fid)
-	if err := c.session.Query(
-		`INSERT INTO seaweed_files (path, fids) VALUES (?, ?)`,
-		fullFileName, input).Exec(); err != nil {
-		glog.V(0).Infof("Failed to save file %s with id %s: %v", fullFileName, fid, err)
-		return err
-	}
-	return nil
-}
-func (c *CassandraStore) Get(fullFileName string) (fid string, err error) {
-	var output []string
-	if err := c.session.Query(
-		`select fids FROM seaweed_files WHERE path = ? LIMIT 1`,
-		fullFileName).Consistency(gocql.One).Scan(&output); err != nil {
-		if err != gocql.ErrNotFound {
-			glog.V(0).Infof("Failed to find file %s: %v", fullFileName, fid, err)
-			return "", filer.ErrNotFound
-		}
-	}
-	if len(output) == 0 {
-		return "", fmt.Errorf("No file id found for %s", fullFileName)
-	}
-	return output[0], nil
-}
-
-// Currently the fid is not returned
-func (c *CassandraStore) Delete(fullFileName string) (err error) {
-	if err := c.session.Query(
-		`DELETE FROM seaweed_files WHERE path = ?`,
-		fullFileName).Exec(); err != nil {
-		if err != gocql.ErrNotFound {
-			glog.V(0).Infof("Failed to delete file %s: %v", fullFileName, err)
-		}
-		return err
-	}
-	return nil
-}
-
-func (c *CassandraStore) Close() {
-	if c.session != nil {
-		c.session.Close()
-	}
-}

weed/filer/cassandra_store/schema.cql

@@ -1,22 +0,0 @@
-/*
-
-Here is the CQL to create the table.CassandraStore
-
-Optionally you can adjust the keyspace name and replication settings.
-
-For production server, very likely you want to set replication_factor to 3
-
-*/
-
-create keyspace seaweed WITH replication = {
-  'class':'SimpleStrategy',
-  'replication_factor':1
-};
-
-use seaweed;
-
-CREATE TABLE seaweed_files (
-   path varchar,
-   fids list<varchar>,
-   PRIMARY KEY (path)
-);

weed/filer/client_operations.go

@@ -1,70 +0,0 @@
-package filer
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-
-	"github.com/chrislusf/seaweedfs/weed/util"
-
-	"net/url"
-)
-
-type ApiRequest struct {
-	Command   string //"listFiles", "listDirectories"
-	Directory string
-	FileName  string
-}
-
-type ListFilesResult struct {
-	Files []FileEntry
-	Error string `json:"error,omitempty"`
-}
-
-func ListFiles(server string, directory string, fileName string) (*ListFilesResult, error) {
-	var ret ListFilesResult
-	if err := call(server, ApiRequest{Command: "listFiles", Directory: directory, FileName: fileName}, &ret); err == nil {
-		if ret.Error != "" {
-			return nil, errors.New(ret.Error)
-		}
-		return &ret, nil
-	} else {
-		return nil, err
-	}
-}
-
-type ListDirectoriesResult struct {
-	Directories []DirectoryEntry
-	Error       string `json:"error,omitempty"`
-}
-
-func ListDirectories(server string, directory string) (*ListDirectoriesResult, error) {
-	var ret ListDirectoriesResult
-	if err := call(server, ApiRequest{Command: "listDirectories", Directory: directory}, &ret); err == nil {
-		if ret.Error != "" {
-			return nil, errors.New(ret.Error)
-		}
-		return &ret, nil
-	} else {
-		return nil, err
-	}
-}
-
-func call(server string, request ApiRequest, ret interface{}) error {
-	b, err := json.Marshal(request)
-	if err != nil {
-		fmt.Println("error:", err)
-		return nil
-	}
-	values := make(url.Values)
-	values.Add("request", string(b))
-	jsonBlob, err := util.Post("http://"+server+"/__api__", values)
-	if err != nil {
-		return err
-	}
-	err = json.Unmarshal(jsonBlob, ret)
-	if err != nil {
-		return err
-	}
-	return nil
-}

weed/filer/embedded_filer/design.txt

@@ -1,26 +0,0 @@
-Design Assumptions:
-1. the number of directories are magnitudely smaller than the number of files
-2. unlimited number of files under any directories
-Phylosophy:
-  metadata for directories and files should be separated
-Design:
-  Store directories in normal map
-    all of directories hopefully all be in memory
-    efficient to move/rename/list_directories
-  Log directory changes to append only log file
-  Store files in sorted string table in <dir_id/filename> format
-    efficient to list_files, just simple iterator
-    efficient to locate files, binary search
-
-Testing:
-1. starting server, "weed server -filer=true"
-2. posting files to different folders
-curl -F "filename=@design.txt"   "http://localhost:8888/sources/"
-curl -F "filename=@design.txt"   "http://localhost:8888/design/"
-curl -F "filename=@directory.go" "http://localhost:8888/sources/weed/go/"
-curl -F "filename=@directory.go" "http://localhost:8888/sources/testing/go/"
-curl -F "filename=@filer.go"     "http://localhost:8888/sources/weed/go/"
-curl -F "filename=@filer_in_leveldb.go"     "http://localhost:8888/sources/weed/go/"
-curl "http://localhost:8888/?pretty=y"
-curl "http://localhost:8888/sources/weed/go/?pretty=y"
-curl "http://localhost:8888/sources/weed/go/?pretty=y"

weed/filer/embedded_filer/directory.go

@@ -1,15 +0,0 @@
-package embedded_filer
-
-import (
-	"github.com/chrislusf/seaweedfs/weed/filer"
-)
-
-type DirectoryManager interface {
-	FindDirectory(dirPath string) (filer.DirectoryId, error)
-	ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error)
-	MakeDirectory(currentDirPath string, dirName string) (filer.DirectoryId, error)
-	MoveUnderDirectory(oldDirPath string, newParentDirPath string) error
-	DeleteDirectory(dirPath string) error
-	//functions used by FUSE
-	FindDirectoryById(filer.DirectoryId, error)
-}

weed/filer/embedded_filer/directory_in_map.go

@@ -1,310 +0,0 @@
-package embedded_filer
-
-import (
-	"bufio"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"sync"
-
-	"github.com/chrislusf/seaweedfs/weed/filer"
-	"github.com/chrislusf/seaweedfs/weed/util"
-)
-
-var writeLock sync.Mutex //serialize changes to dir.log
-
-type DirectoryEntryInMap struct {
-	sync.Mutex
-	Name           string
-	Parent         *DirectoryEntryInMap
-	subDirectories map[string]*DirectoryEntryInMap
-	Id             filer.DirectoryId
-}
-
-func (de *DirectoryEntryInMap) getChild(dirName string) (*DirectoryEntryInMap, bool) {
-	de.Lock()
-	defer de.Unlock()
-	child, ok := de.subDirectories[dirName]
-	return child, ok
-}
-func (de *DirectoryEntryInMap) addChild(dirName string, child *DirectoryEntryInMap) {
-	de.Lock()
-	defer de.Unlock()
-	de.subDirectories[dirName] = child
-}
-func (de *DirectoryEntryInMap) removeChild(dirName string) {
-	de.Lock()
-	defer de.Unlock()
-	delete(de.subDirectories, dirName)
-}
-func (de *DirectoryEntryInMap) hasChildren() bool {
-	de.Lock()
-	defer de.Unlock()
-	return len(de.subDirectories) > 0
-}
-func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryEntry) {
-	de.Lock()
-	defer de.Unlock()
-	for k, v := range de.subDirectories {
-		dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id})
-	}
-	return dirNames
-}
-
-type DirectoryManagerInMap struct {
-	Root      *DirectoryEntryInMap
-	max       filer.DirectoryId
-	logFile   *os.File
-	isLoading bool
-}
-
-func (dm *DirectoryManagerInMap) newDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap, err error) {
-	d = &DirectoryEntryInMap{Name: name, Parent: parent, subDirectories: make(map[string]*DirectoryEntryInMap)}
-	var parts []string
-	for p := d; p != nil && p.Name != ""; p = p.Parent {
-		parts = append(parts, p.Name)
-	}
-	n := len(parts)
-	if n <= 0 {
-		return nil, fmt.Errorf("Failed to create folder %s/%s", parent.Name, name)
-	}
-	for i := 0; i < n/2; i++ {
-		parts[i], parts[n-1-i] = parts[n-1-i], parts[i]
-	}
-	dm.max++
-	d.Id = dm.max
-	dm.log("add", "/"+strings.Join(parts, "/"), strconv.Itoa(int(d.Id)))
-	return d, nil
-}
-
-func (dm *DirectoryManagerInMap) log(words ...string) {
-	if !dm.isLoading {
-		dm.logFile.WriteString(strings.Join(words, "\t") + "\n")
-	}
-}
-
-func NewDirectoryManagerInMap(dirLogFile string) (dm *DirectoryManagerInMap, err error) {
-	dm = &DirectoryManagerInMap{}
-	//dm.Root do not use newDirectoryEntryInMap, since dm.max will be changed
-	dm.Root = &DirectoryEntryInMap{subDirectories: make(map[string]*DirectoryEntryInMap)}
-	if dm.logFile, err = os.OpenFile(dirLogFile, os.O_RDWR|os.O_CREATE, 0644); err != nil {
-		return nil, fmt.Errorf("cannot write directory log file %s: %v", dirLogFile, err)
-	}
-	return dm, dm.load()
-}
-
-func (dm *DirectoryManagerInMap) processEachLine(line string) error {
-	if strings.HasPrefix(line, "#") {
-		return nil
-	}
-	if line == "" {
-		return nil
-	}
-	parts := strings.Split(line, "\t")
-	if len(parts) == 0 {
-		return nil
-	}
-	switch parts[0] {
-	case "add":
-		v, pe := strconv.Atoi(parts[2])
-		if pe != nil {
-			return pe
-		}
-		if e := dm.loadDirectory(parts[1], filer.DirectoryId(v)); e != nil {
-			return e
-		}
-	case "mov":
-		newName := ""
-		if len(parts) >= 4 {
-			newName = parts[3]
-		}
-		if e := dm.MoveUnderDirectory(parts[1], parts[2], newName); e != nil {
-			return e
-		}
-	case "del":
-		if e := dm.DeleteDirectory(parts[1]); e != nil {
-			return e
-		}
-	default:
-		fmt.Printf("line %s has %s!\n", line, parts[0])
-		return nil
-	}
-	return nil
-}
-func (dm *DirectoryManagerInMap) load() error {
-	dm.max = 0
-	lines := bufio.NewReader(dm.logFile)
-	dm.isLoading = true
-	defer func() { dm.isLoading = false }()
-	for {
-		line, err := util.Readln(lines)
-		if err != nil && err != io.EOF {
-			return err
-		}
-		if pe := dm.processEachLine(string(line)); pe != nil {
-			return pe
-		}
-		if err == io.EOF {
-			return nil
-		}
-	}
-}
-
-func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryInMap, error) {
-	if dirPath == "" {
-		return dm.Root, nil
-	}
-	dirPath = CleanFilePath(dirPath)
-	if dirPath == "/" {
-		return dm.Root, nil
-	}
-	parts := strings.Split(dirPath, "/")
-	dir := dm.Root
-	for i := 1; i < len(parts); i++ {
-		if sub, ok := dir.getChild(parts[i]); ok {
-			dir = sub
-		} else {
-			return dm.Root, fmt.Errorf("Directory %s Not Found", dirPath)
-		}
-	}
-	return dir, nil
-}
-func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryId, error) {
-	d, e := dm.findDirectory(dirPath)
-	if e == nil {
-		return d.Id, nil
-	}
-	return dm.Root.Id, e
-}
-
-func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error {
-	dirPath = CleanFilePath(dirPath)
-	if dirPath == "/" {
-		return nil
-	}
-	parts := strings.Split(dirPath, "/")
-	dir := dm.Root
-	for i := 1; i < len(parts); i++ {
-		sub, ok := dir.getChild(parts[i])
-		if !ok {
-			writeLock.Lock()
-			if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread {
-				sub = sub2
-			} else {
-				if i != len(parts)-1 {
-					writeLock.Unlock()
-					return fmt.Errorf("%s should be created after parent %s", dirPath, parts[i])
-				}
-				var err error
-				sub, err = dm.newDirectoryEntryInMap(dir, parts[i])
-				if err != nil {
-					writeLock.Unlock()
-					return err
-				}
-				if sub.Id != dirId {
-					writeLock.Unlock()
-					// the dir.log should be the same order as in-memory directory id
-					return fmt.Errorf("%s should be have id %v instead of %v", dirPath, sub.Id, dirId)
-				}
-				dir.addChild(parts[i], sub)
-			}
-			writeLock.Unlock()
-		}
-		dir = sub
-	}
-	return nil
-}
-
-func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEntryInMap, created bool) {
-	dirPath = CleanFilePath(dirPath)
-	if dirPath == "/" {
-		return dm.Root, false
-	}
-	parts := strings.Split(dirPath, "/")
-	dir = dm.Root
-	for i := 1; i < len(parts); i++ {
-		sub, ok := dir.getChild(parts[i])
-		if !ok {
-			writeLock.Lock()
-			if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread {
-				sub = sub2
-			} else {
-				var err error
-				sub, err = dm.newDirectoryEntryInMap(dir, parts[i])
-				if err != nil {
-					writeLock.Unlock()
-					return nil, false
-				}
-				dir.addChild(parts[i], sub)
-				created = true
-			}
-			writeLock.Unlock()
-		}
-		dir = sub
-	}
-	return dir, created
-}
-
-func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (filer.DirectoryId, error) {
-	dir, _ := dm.makeDirectory(dirPath)
-	return dir.Id, nil
-}
-
-func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParentDirPath string, newName string) error {
-	writeLock.Lock()
-	defer writeLock.Unlock()
-	oldDir, oe := dm.findDirectory(oldDirPath)
-	if oe != nil {
-		return oe
-	}
-	parentDir, pe := dm.findDirectory(newParentDirPath)
-	if pe != nil {
-		return pe
-	}
-	dm.log("mov", oldDirPath, newParentDirPath, newName)
-	oldDir.Parent.removeChild(oldDir.Name)
-	if newName == "" {
-		newName = oldDir.Name
-	}
-	parentDir.addChild(newName, oldDir)
-	oldDir.Name = newName
-	oldDir.Parent = parentDir
-	return nil
-}
-
-func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryEntry, err error) {
-	d, e := dm.findDirectory(dirPath)
-	if e != nil {
-		return dirNames, e
-	}
-	return d.children(), nil
-}
-func (dm *DirectoryManagerInMap) DeleteDirectory(dirPath string) error {
-	writeLock.Lock()
-	defer writeLock.Unlock()
-	if dirPath == "/" {
-		return fmt.Errorf("Can not delete %s", dirPath)
-	}
-	d, e := dm.findDirectory(dirPath)
-	if e != nil {
-		return e
-	}
-	if d.hasChildren() {
-		return fmt.Errorf("dir %s still has sub directories", dirPath)
-	}
-	d.Parent.removeChild(d.Name)
-	d.Parent = nil
-	dm.log("del", dirPath)
-	return nil
-}
-
-func CleanFilePath(fp string) string {
-	ret := filepath.Clean(fp)
-	if os.PathSeparator == '\\' {
-		return strings.Replace(ret, "\\", "/", -1)
-	}
-	return ret
-}

weed/filer/embedded_filer/directory_test.go

@@ -1,86 +0,0 @@
-package embedded_filer
-
-import (
-	"os"
-	"strings"
-	"testing"
-)
-
-func TestDirectory(t *testing.T) {
-	dm, _ := NewDirectoryManagerInMap("/tmp/dir.log")
-	defer func() {
-		if true {
-			os.Remove("/tmp/dir.log")
-		}
-	}()
-	dm.MakeDirectory("/a/b/c")
-	dm.MakeDirectory("/a/b/d")
-	dm.MakeDirectory("/a/b/e")
-	dm.MakeDirectory("/a/b/e/f")
-	dm.MakeDirectory("/a/b/e/f/g")
-	dm.MoveUnderDirectory("/a/b/e/f/g", "/a/b", "t")
-	if _, err := dm.FindDirectory("/a/b/e/f/g"); err == nil {
-		t.Fatal("/a/b/e/f/g should not exist any more after moving")
-	}
-	if _, err := dm.FindDirectory("/a/b/t"); err != nil {
-		t.Fatal("/a/b/t should exist after moving")
-	}
-	if _, err := dm.FindDirectory("/a/b/g"); err == nil {
-		t.Fatal("/a/b/g should not exist after moving")
-	}
-	dm.MoveUnderDirectory("/a/b/e/f", "/a/b", "")
-	if _, err := dm.FindDirectory("/a/b/f"); err != nil {
-		t.Fatal("/a/b/g should not exist after moving")
-	}
-	dm.MakeDirectory("/a/b/g/h/i")
-	dm.DeleteDirectory("/a/b/e/f")
-	dm.DeleteDirectory("/a/b/e")
-	dirNames, _ := dm.ListDirectories("/a/b/e")
-	for _, v := range dirNames {
-		println("sub1 dir:", v.Name, "id", v.Id)
-	}
-	dm.logFile.Close()
-
-	var path []string
-	printTree(dm.Root, path)
-
-	dm2, e := NewDirectoryManagerInMap("/tmp/dir.log")
-	if e != nil {
-		println("load error", e.Error())
-	}
-	if !compare(dm.Root, dm2.Root) {
-		t.Fatal("restored dir not the same!")
-	}
-	printTree(dm2.Root, path)
-}
-
-func printTree(node *DirectoryEntryInMap, path []string) {
-	println(strings.Join(path, "/") + "/" + node.Name)
-	path = append(path, node.Name)
-	for _, v := range node.subDirectories {
-		printTree(v, path)
-	}
-}
-
-func compare(root1 *DirectoryEntryInMap, root2 *DirectoryEntryInMap) bool {
-	if len(root1.subDirectories) != len(root2.subDirectories) {
-		return false
-	}
-	if root1.Name != root2.Name {
-		return false
-	}
-	if root1.Id != root2.Id {
-		return false
-	}
-	if !(root1.Parent == nil && root2.Parent == nil) {
-		if root1.Parent.Id != root2.Parent.Id {
-			return false
-		}
-	}
-	for k, v := range root1.subDirectories {
-		if !compare(v, root2.subDirectories[k]) {
-			return false
-		}
-	}
-	return true
-}

weed/filer/embedded_filer/filer_embedded.go

@@ -1,141 +0,0 @@
-package embedded_filer
-
-import (
-	"errors"
-	"fmt"
-	"path/filepath"
-	"strings"
-
-	"github.com/chrislusf/seaweedfs/weed/filer"
-	"github.com/chrislusf/seaweedfs/weed/operation"
-)
-
-type FilerEmbedded struct {
-	master      string
-	directories *DirectoryManagerInMap
-	files       *FileListInLevelDb
-}
-
-func NewFilerEmbedded(master string, dir string) (filer *FilerEmbedded, err error) {
-	dm, de := NewDirectoryManagerInMap(filepath.Join(dir, "dir.log"))
-	if de != nil {
-		return nil, de
-	}
-	fl, fe := NewFileListInLevelDb(dir)
-	if fe != nil {
-		return nil, fe
-	}
-	filer = &FilerEmbedded{
-		master:      master,
-		directories: dm,
-		files:       fl,
-	}
-	return
-}
-
-func (filer *FilerEmbedded) CreateFile(filePath string, fid string) (err error) {
-	dir, file := filepath.Split(filePath)
-	dirId, e := filer.directories.MakeDirectory(dir)
-	if e != nil {
-		return e
-	}
-	return filer.files.CreateFile(dirId, file, fid)
-}
-func (filer *FilerEmbedded) FindFile(filePath string) (fid string, err error) {
-	dir, file := filepath.Split(filePath)
-	dirId, e := filer.directories.FindDirectory(dir)
-	if e != nil {
-		return "", e
-	}
-	return filer.files.FindFile(dirId, file)
-}
-func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
-	return filer.directories.FindDirectory(dirPath)
-}
-func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
-	return filer.directories.ListDirectories(dirPath)
-}
-func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
-	dirId, e := filer.directories.FindDirectory(dirPath)
-	if e != nil {
-		return nil, e
-	}
-	return filer.files.ListFiles(dirId, lastFileName, limit), nil
-}
-func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err error) {
-	dirId, e := filer.directories.FindDirectory(dirPath)
-	if e != nil {
-		return e
-	}
-	if sub_dirs, sub_err := filer.directories.ListDirectories(dirPath); sub_err == nil {
-		if len(sub_dirs) > 0 && !recursive {
-			return fmt.Errorf("Fail to delete directory %s: %d sub directories found!", dirPath, len(sub_dirs))
-		}
-		for _, sub := range sub_dirs {
-			if delete_sub_err := filer.DeleteDirectory(filepath.Join(dirPath, sub.Name), recursive); delete_sub_err != nil {
-				return delete_sub_err
-			}
-		}
-	}
-	list := filer.files.ListFiles(dirId, "", 100)
-	if len(list) != 0 && !recursive {
-		if !recursive {
-			return fmt.Errorf("Fail to delete non-empty directory %s!", dirPath)
-		}
-	}
-	for {
-		if len(list) == 0 {
-			return filer.directories.DeleteDirectory(dirPath)
-		}
-		var fids []string
-		for _, fileEntry := range list {
-			fids = append(fids, string(fileEntry.Id))
-		}
-		if result_list, delete_file_err := operation.DeleteFiles(filer.master, fids); delete_file_err != nil {
-			return delete_file_err
-		} else {
-			if len(result_list.Errors) > 0 {
-				return errors.New(strings.Join(result_list.Errors, "\n"))
-			}
-		}
-		lastFile := list[len(list)-1]
-		list = filer.files.ListFiles(dirId, lastFile.Name, 100)
-	}
-
-}
-
-func (filer *FilerEmbedded) DeleteFile(filePath string) (fid string, err error) {
-	dir, file := filepath.Split(filePath)
-	dirId, e := filer.directories.FindDirectory(dir)
-	if e != nil {
-		return "", e
-	}
-	return filer.files.DeleteFile(dirId, file)
-}
-
-/*
-Move a folder or a file, with 4 Use cases:
-mv fromDir toNewDir
-mv fromDir toOldDir
-mv fromFile toDir
-mv fromFile toFile
-*/
-func (filer *FilerEmbedded) Move(fromPath string, toPath string) error {
-	if _, dir_err := filer.FindDirectory(fromPath); dir_err == nil {
-		if _, err := filer.FindDirectory(toPath); err == nil {
-			// move folder under an existing folder
-			return filer.directories.MoveUnderDirectory(fromPath, toPath, "")
-		}
-		// move folder to a new folder
-		return filer.directories.MoveUnderDirectory(fromPath, filepath.Dir(toPath), filepath.Base(toPath))
-	}
-	if fid, file_err := filer.DeleteFile(fromPath); file_err == nil {
-		if _, err := filer.FindDirectory(toPath); err == nil {
-			// move file under an existing folder
-			return filer.CreateFile(filepath.Join(toPath, filepath.Base(fromPath)), fid)
-		}
-		// move to a folder with new name
-		return filer.CreateFile(toPath, fid)
-	}
-	return fmt.Errorf("File %s is not found!", fromPath)
-}

weed/filer/embedded_filer/files_in_leveldb.go

@@ -1,87 +0,0 @@
-package embedded_filer
-
-import (
-	"bytes"
-
-	"github.com/chrislusf/seaweedfs/weed/filer"
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-/*
-The entry in level db has this format:
-  key: genKey(dirId, fileName)
-  value: []byte(fid)
-And genKey(dirId, fileName) use first 4 bytes to store dirId, and rest for fileName
-*/
-
-type FileListInLevelDb struct {
-	db *leveldb.DB
-}
-
-func NewFileListInLevelDb(dir string) (fl *FileListInLevelDb, err error) {
-	fl = &FileListInLevelDb{}
-	if fl.db, err = leveldb.OpenFile(dir, nil); err != nil {
-		return
-	}
-	return
-}
-
-func genKey(dirId filer.DirectoryId, fileName string) []byte {
-	ret := make([]byte, 0, 4+len(fileName))
-	for i := 3; i >= 0; i-- {
-		ret = append(ret, byte(dirId>>(uint(i)*8)))
-	}
-	ret = append(ret, []byte(fileName)...)
-	return ret
-}
-
-func (fl *FileListInLevelDb) CreateFile(dirId filer.DirectoryId, fileName string, fid string) (err error) {
-	glog.V(4).Infoln("directory", dirId, "fileName", fileName, "fid", fid)
-	return fl.db.Put(genKey(dirId, fileName), []byte(fid), nil)
-}
-func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
-	if fid, err = fl.FindFile(dirId, fileName); err != nil {
-		if err == leveldb.ErrNotFound {
-			return "", nil
-		}
-		return
-	}
-	err = fl.db.Delete(genKey(dirId, fileName), nil)
-	return fid, err
-}
-func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
-	data, e := fl.db.Get(genKey(dirId, fileName), nil)
-	if e == leveldb.ErrNotFound {
-		return "", filer.ErrNotFound
-	} else if e != nil {
-		return "", e
-	}
-	return string(data), nil
-}
-func (fl *FileListInLevelDb) ListFiles(dirId filer.DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) {
-	glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit)
-	dirKey := genKey(dirId, "")
-	iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil)
-	limitCounter := 0
-	for iter.Next() {
-		key := iter.Key()
-		if !bytes.HasPrefix(key, dirKey) {
-			break
-		}
-		fileName := string(key[len(dirKey):])
-		if fileName == lastFileName {
-			continue
-		}
-		limitCounter++
-		if limit > 0 {
-			if limitCounter > limit {
-				break
-			}
-		}
-		files = append(files, filer.FileEntry{Name: fileName, Id: filer.FileId(string(iter.Value()))})
-	}
-	iter.Release()
-	return
-}

weed/filer/filer.go

@@ -1,34 +0,0 @@
-package filer
-
-import (
-	"errors"
-)
-
-type FileId string //file id in SeaweedFS
-
-type FileEntry struct {
-	Name string `json:"name,omitempty"` //file name without path
-	Id   FileId `json:"fid,omitempty"`
-}
-
-type DirectoryId int32
-
-type DirectoryEntry struct {
-	Name string //dir name without path
-	Id   DirectoryId
-}
-
-type Filer interface {
-	CreateFile(fullFileName string, fid string) (err error)
-	FindFile(fullFileName string) (fid string, err error)
-	DeleteFile(fullFileName string) (fid string, err error)
-
-	//Optional functions. embedded filer support these
-	FindDirectory(dirPath string) (dirId DirectoryId, err error)
-	ListDirectories(dirPath string) (dirs []DirectoryEntry, err error)
-	ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error)
-	DeleteDirectory(dirPath string, recursive bool) (err error)
-	Move(fromPath string, toPath string) (err error)
-}
-
-var ErrNotFound = errors.New("filer: no entry is found in filer store")

weed/filer/flat_namespace/flat_namespace_filer.go

@@ -1,62 +0,0 @@
-package flat_namespace
-
-import (
-	"errors"
-
-	"github.com/chrislusf/seaweedfs/weed/filer"
-)
-
-type FlatNamespaceFiler struct {
-	master string
-	store  FlatNamespaceStore
-}
-
-var (
-	ErrNotImplemented = errors.New("Not Implemented for flat namespace meta data store")
-)
-
-func NewFlatNamespaceFiler(master string, store FlatNamespaceStore) *FlatNamespaceFiler {
-	return &FlatNamespaceFiler{
-		master: master,
-		store:  store,
-	}
-}
-
-func (filer *FlatNamespaceFiler) CreateFile(fullFileName string, fid string) (err error) {
-	return filer.store.Put(fullFileName, fid)
-}
-func (filer *FlatNamespaceFiler) FindFile(fullFileName string) (fid string, err error) {
-	return filer.store.Get(fullFileName)
-}
-func (filer *FlatNamespaceFiler) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
-	return 0, ErrNotImplemented
-}
-func (filer *FlatNamespaceFiler) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
-	return nil, ErrNotImplemented
-}
-func (filer *FlatNamespaceFiler) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
-	return nil, ErrNotImplemented
-}
-func (filer *FlatNamespaceFiler) DeleteDirectory(dirPath string, recursive bool) (err error) {
-	return ErrNotImplemented
-}
-
-func (filer *FlatNamespaceFiler) DeleteFile(fullFileName string) (fid string, err error) {
-	fid, err = filer.FindFile(fullFileName)
-	if err != nil {
-		return "", err
-	}
-
-	err = filer.store.Delete(fullFileName)
-	if err != nil {
-		return "", err
-	}
-
-	return fid, nil
-	//return filer.store.Delete(fullFileName)
-	//are you kidding me!!!!
-}
-
-func (filer *FlatNamespaceFiler) Move(fromPath string, toPath string) error {
-	return ErrNotImplemented
-}

weed/filer/flat_namespace/flat_namespace_store.go

@@ -1,9 +0,0 @@
-package flat_namespace
-
-import ()
-
-type FlatNamespaceStore interface {
-	Put(fullFileName string, fid string) (err error)
-	Get(fullFileName string) (fid string, err error)
-	Delete(fullFileName string) (err error)
-}

weed/filer/mysql_store/README.md

@@ -1,67 +0,0 @@
-#MySQL filer mapping store
-
-## Schema format
-
-
-Basically, uriPath and fid are the key elements stored in MySQL. In view of the optimization and user's usage, 
-adding primary key with integer type and involving createTime, updateTime, status fields should be somewhat meaningful.
-Of course, you could customize the schema per your concretely circumstance freely.
-
-<pre><code>
-CREATE TABLE IF NOT EXISTS `filer_mapping` (
-  `id` bigint(20) NOT NULL AUTO_INCREMENT,
-  `uriPath` char(256) NOT NULL DEFAULT "" COMMENT 'http uriPath',
-  `fid` char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid',
-  `createTime` int(10) NOT NULL DEFAULT 0 COMMENT 'createdTime in unix timestamp',
-  `updateTime` int(10) NOT NULL DEFAULT 0 COMMENT 'updatedTime in unix timestamp',
-  `remark` varchar(20) NOT NULL DEFAULT "" COMMENT 'reserverd field',
-  `status` tinyint(2) DEFAULT '1' COMMENT 'resource status',
-  PRIMARY KEY (`id`),
-  UNIQUE KEY `index_uriPath` (`uriPath`)
-) DEFAULT CHARSET=utf8;
-</code></pre>
-
-
-The MySQL 's config params is not added into the weed command option as other stores(redis,cassandra). Instead,
-We created a config file(json format) for them. TOML,YAML or XML also should be OK. But TOML and YAML need import thirdparty package
-while XML is a little bit complex.
-
-The sample config file's  content is below:
-
-<pre><code>
-{
-    "mysql": [
-        {
-            "User": "root",
-            "Password": "root",
-            "HostName": "127.0.0.1",
-            "Port": 3306,
-            "DataBase": "seaweedfs"
-        },
-        {
-            "User": "root",
-            "Password": "root",
-            "HostName": "127.0.0.2",
-            "Port": 3306,
-            "DataBase": "seaweedfs"
-        }
-    ],
-    "IsSharding":true,
-    "ShardCount":1024
-}
-</code></pre>
-
-
-The "mysql" field in above conf file is an array which include all mysql instances you prepared to store sharding data.
-
-1. If one mysql instance is enough, just keep one instance in "mysql" field.
-
-2. If table sharding at a specific mysql instance is needed , mark "IsSharding" field with true and specify total table sharding numbers using "ShardCount" field.
-
-3. If the mysql service could be auto scaled transparently in your environment, just config one mysql instance(usually it's a frondend proxy or VIP),and mark "IsSharding" with false value
-
-4. If you prepare more than one mysql instance and have no plan to use table sharding for any instance(mark isSharding with false), instance sharding will still be done implicitly
-
-
-
-

weed/filer/mysql_store/mysql_store.go

@@ -1,270 +0,0 @@
-package mysql_store
-
-import (
-	"database/sql"
-	"fmt"
-	"hash/crc32"
-	"sync"
-	"time"
-
-	"github.com/chrislusf/seaweedfs/weed/filer"
-
-	_ "github.com/go-sql-driver/mysql"
-)
-
-const (
-	sqlUrl                     = "%s:%s@tcp(%s:%d)/%s?charset=utf8"
-	default_maxIdleConnections = 100
-	default_maxOpenConnections = 50
-	default_maxTableNums       = 1024
-	tableName                  = "filer_mapping"
-)
-
-var (
-	_init_db        sync.Once
-	_db_connections []*sql.DB
-)
-
-type MySqlConf struct {
-	User               string
-	Password           string
-	HostName           string
-	Port               int
-	DataBase           string
-	MaxIdleConnections int
-	MaxOpenConnections int
-}
-
-type ShardingConf struct {
-	IsSharding bool `json:"isSharding"`
-	ShardCount int  `json:"shardCount"`
-}
-
-type MySqlStore struct {
-	dbs        []*sql.DB
-	isSharding bool
-	shardCount int
-}
-
-func getDbConnection(confs []MySqlConf) []*sql.DB {
-	_init_db.Do(func() {
-		for _, conf := range confs {
-
-			sqlUrl := fmt.Sprintf(sqlUrl, conf.User, conf.Password, conf.HostName, conf.Port, conf.DataBase)
-			var dbErr error
-			_db_connection, dbErr := sql.Open("mysql", sqlUrl)
-			if dbErr != nil {
-				_db_connection.Close()
-				_db_connection = nil
-				panic(dbErr)
-			}
-			var maxIdleConnections, maxOpenConnections int
-
-			if conf.MaxIdleConnections != 0 {
-				maxIdleConnections = conf.MaxIdleConnections
-			} else {
-				maxIdleConnections = default_maxIdleConnections
-			}
-			if conf.MaxOpenConnections != 0 {
-				maxOpenConnections = conf.MaxOpenConnections
-			} else {
-				maxOpenConnections = default_maxOpenConnections
-			}
-
-			_db_connection.SetMaxIdleConns(maxIdleConnections)
-			_db_connection.SetMaxOpenConns(maxOpenConnections)
-			_db_connections = append(_db_connections, _db_connection)
-		}
-	})
-	return _db_connections
-}
-
-func NewMysqlStore(confs []MySqlConf, isSharding bool, shardCount int) *MySqlStore {
-	ms := &MySqlStore{
-		dbs:        getDbConnection(confs),
-		isSharding: isSharding,
-		shardCount: shardCount,
-	}
-
-	for _, db := range ms.dbs {
-		if !isSharding {
-			ms.shardCount = 1
-		} else {
-			if ms.shardCount == 0 {
-				ms.shardCount = default_maxTableNums
-			}
-		}
-		for i := 0; i < ms.shardCount; i++ {
-			if err := ms.createTables(db, tableName, i); err != nil {
-				fmt.Printf("create table failed %v", err)
-			}
-		}
-	}
-
-	return ms
-}
-
-func (s *MySqlStore) hash(fullFileName string) (instance_offset, table_postfix int) {
-	hash_value := crc32.ChecksumIEEE([]byte(fullFileName))
-	instance_offset = int(hash_value) % len(s.dbs)
-	table_postfix = int(hash_value) % s.shardCount
-	return
-}
-
-func (s *MySqlStore) parseFilerMappingInfo(path string) (instanceId int, tableFullName string, err error) {
-	instance_offset, table_postfix := s.hash(path)
-	instanceId = instance_offset
-	if s.isSharding {
-		tableFullName = fmt.Sprintf("%s_%04d", tableName, table_postfix)
-	} else {
-		tableFullName = tableName
-	}
-	return
-}
-
-func (s *MySqlStore) Get(fullFilePath string) (fid string, err error) {
-	instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath)
-	if err != nil {
-		return "", fmt.Errorf("MySqlStore Get operation can not parse file path %s: err is %v", fullFilePath, err)
-	}
-	fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName)
-	if err == sql.ErrNoRows {
-		//Could not found
-		err = filer.ErrNotFound
-	}
-	return fid, err
-}
-
-func (s *MySqlStore) Put(fullFilePath string, fid string) (err error) {
-	var tableFullName string
-
-	instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath)
-	if err != nil {
-		return fmt.Errorf("MySqlStore Put operation can not parse file path %s: err is %v", fullFilePath, err)
-	}
-	var old_fid string
-	if old_fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil && err != sql.ErrNoRows {
-		return fmt.Errorf("MySqlStore Put operation failed when querying path %s: err is %v", fullFilePath, err)
-	} else {
-		if len(old_fid) == 0 {
-			err = s.insert(fullFilePath, fid, s.dbs[instance_offset], tableFullName)
-			err = fmt.Errorf("MySqlStore Put operation failed when inserting path %s with fid %s : err is %v", fullFilePath, fid, err)
-		} else {
-			err = s.update(fullFilePath, fid, s.dbs[instance_offset], tableFullName)
-			err = fmt.Errorf("MySqlStore Put operation failed when updating path %s with fid %s : err is %v", fullFilePath, fid, err)
-		}
-	}
-	return
-}
-
-func (s *MySqlStore) Delete(fullFilePath string) (err error) {
-	var fid string
-	instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath)
-	if err != nil {
-		return fmt.Errorf("MySqlStore Delete operation can not parse file path %s: err is %v", fullFilePath, err)
-	}
-	if fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil {
-		return fmt.Errorf("MySqlStore Delete operation failed when querying path %s: err is %v", fullFilePath, err)
-	} else if fid == "" {
-		return nil
-	}
-	if err = s.delete(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil {
-		return fmt.Errorf("MySqlStore Delete operation failed when deleting path %s: err is %v", fullFilePath, err)
-	} else {
-		return nil
-	}
-}
-
-func (s *MySqlStore) Close() {
-	for _, db := range s.dbs {
-		db.Close()
-	}
-}
-
-var createTable = `
-CREATE TABLE IF NOT EXISTS %s (
-  id bigint(20) NOT NULL AUTO_INCREMENT,
-  uriPath char(256) NOT NULL DEFAULT "" COMMENT 'http uriPath',
-  fid char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid',
-  createTime int(10) NOT NULL DEFAULT 0 COMMENT 'createdTime in unix timestamp',
-  updateTime int(10) NOT NULL DEFAULT 0 COMMENT 'updatedTime in unix timestamp',
-  remark varchar(20) NOT NULL DEFAULT "" COMMENT 'reserverd field',
-  status tinyint(2) DEFAULT '1' COMMENT 'resource status',
-  PRIMARY KEY (id),
-  UNIQUE KEY index_uriPath (uriPath)
-) DEFAULT CHARSET=utf8;
-`
-
-func (s *MySqlStore) createTables(db *sql.DB, tableName string, postfix int) error {
-	var realTableName string
-	if s.isSharding {
-		realTableName = fmt.Sprintf("%s_%4d", tableName, postfix)
-	} else {
-		realTableName = tableName
-	}
-
-	stmt, err := db.Prepare(fmt.Sprintf(createTable, realTableName))
-	if err != nil {
-		return err
-	}
-	defer stmt.Close()
-
-	_, err = stmt.Exec()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *MySqlStore) query(uriPath string, db *sql.DB, tableName string) (string, error) {
-	sqlStatement := "SELECT fid FROM %s WHERE uriPath=?"
-	row := db.QueryRow(fmt.Sprintf(sqlStatement, tableName), uriPath)
-	var fid string
-	err := row.Scan(&fid)
-	if err != nil {
-		return "", err
-	}
-	return fid, nil
-}
-
-func (s *MySqlStore) update(uriPath string, fid string, db *sql.DB, tableName string) error {
-	sqlStatement := "UPDATE %s SET fid=?, updateTime=? WHERE uriPath=?"
-	res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), fid, time.Now().Unix(), uriPath)
-	if err != nil {
-		return err
-	}
-
-	_, err = res.RowsAffected()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *MySqlStore) insert(uriPath string, fid string, db *sql.DB, tableName string) error {
-	sqlStatement := "INSERT INTO %s (uriPath,fid,createTime) VALUES(?,?,?)"
-	res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), uriPath, fid, time.Now().Unix())
-	if err != nil {
-		return err
-	}
-
-	_, err = res.RowsAffected()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *MySqlStore) delete(uriPath string, db *sql.DB, tableName string) error {
-	sqlStatement := "DELETE FROM %s WHERE uriPath=?"
-	res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), uriPath)
-	if err != nil {
-		return err
-	}
-
-	_, err = res.RowsAffected()
-	if err != nil {
-		return err
-	}
-	return nil
-}

weed/filer/mysql_store/mysql_store_test.go

@@ -1,30 +0,0 @@
-package mysql_store
-
-import (
-	"encoding/json"
-	"hash/crc32"
-	"testing"
-)
-
-func TestGenerateMysqlConf(t *testing.T) {
-	var conf []MySqlConf
-	conf = append(conf, MySqlConf{
-		User:     "root",
-		Password: "root",
-		HostName: "localhost",
-		Port:     3306,
-		DataBase: "seaweedfs",
-	})
-	body, err := json.Marshal(conf)
-	if err != nil {
-		t.Errorf("json encoding err %s", err.Error())
-	}
-	t.Logf("json output is %s", string(body))
-}
-
-func TestCRC32FullPathName(t *testing.T) {
-	fullPathName := "/prod-bucket/law632191483895612493300-signed.pdf"
-	hash_value := crc32.ChecksumIEEE([]byte(fullPathName))
-	table_postfix := int(hash_value) % 1024
-	t.Logf("table postfix %d", table_postfix)
-}

weed/filer/postgres_store/postgres_store.go

@@ -1,623 +0,0 @@
-package postgres_store
-
-import (
-	"database/sql"
-	"errors"
-	"fmt"
-	"path/filepath"
-	"sync"
-	"time"
-
-	"github.com/chrislusf/seaweedfs/weed/filer"
-	"github.com/chrislusf/seaweedfs/weed/glog"
-
-	_ "github.com/lib/pq"
-	_ "path/filepath"
-	"strings"
-)
-
-const (
-	default_maxIdleConnections = 100
-	default_maxOpenConnections = 50
-	filesTableName             = "files"
-	directoriesTableName       = "directories"
-)
-
-var (
-	_init_db       sync.Once
-	_db_connection *sql.DB
-)
-
-type PostgresConf struct {
-	User               string
-	Password           string
-	HostName           string
-	Port               int
-	DataBase           string
-	SslMode            string
-	MaxIdleConnections int
-	MaxOpenConnections int
-}
-
-type PostgresStore struct {
-	db       *sql.DB
-	server   string
-	user     string
-	password string
-}
-
-func (s *PostgresStore) CreateFile(fullFileName string, fid string) (err error) {
-	glog.V(3).Infoln("Calling posgres_store CreateFile")
-	return s.Put(fullFileName, fid)
-}
-
-func (s *PostgresStore) FindFile(fullFileName string) (fid string, err error) {
-	glog.V(3).Infoln("Calling posgres_store FindFile")
-	return s.Get(fullFileName)
-}
-
-func (s *PostgresStore) DeleteFile(fullFileName string) (fid string, err error) {
-	glog.V(3).Infoln("Calling posgres_store DeleteFile")
-	return "", s.Delete(fullFileName)
-}
-
-func (s *PostgresStore) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
-	glog.V(3).Infoln("Calling posgres_store FindDirectory")
-	return s.FindDir(dirPath)
-}
-
-func (s *PostgresStore) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
-	glog.V(3).Infoln("Calling posgres_store ListDirectories")
-	return s.ListDirs(dirPath)
-}
-
-func (s *PostgresStore) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
-	glog.V(3).Infoln("Calling posgres_store ListFiles")
-	return s.FindFiles(dirPath, lastFileName, limit)
-}
-
-func (s *PostgresStore) DeleteDirectory(dirPath string, recursive bool) (err error) {
-	glog.V(3).Infoln("Calling posgres_store DeleteDirectory")
-	return s.DeleteDir(dirPath, recursive)
-}
-
-func (s *PostgresStore) Move(fromPath string, toPath string) (err error) {
-	glog.V(3).Infoln("Calling posgres_store Move")
-	return errors.New("Move is not yet implemented for the PostgreSQL store.")
-}
-
-func databaseExists(db *sql.DB, databaseName string) (bool, error) {
-	sqlStatement := "SELECT datname from pg_database WHERE datname='%s'"
-	row := db.QueryRow(fmt.Sprintf(sqlStatement, databaseName))
-
-	var dbName string
-	err := row.Scan(&dbName)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return false, nil
-		}
-		return false, err
-	}
-	return true, nil
-}
-
-func createDatabase(db *sql.DB, databaseName string) error {
-	sqlStatement := "CREATE DATABASE %s ENCODING='UTF8'"
-	_, err := db.Exec(fmt.Sprintf(sqlStatement, databaseName))
-	return err
-}
-
-func getDbConnection(conf PostgresConf) *sql.DB {
-	_init_db.Do(func() {
-
-		sqlUrl := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30", conf.HostName, conf.Port, conf.User, conf.Password, "postgres", conf.SslMode)
-		glog.V(3).Infoln("Opening postgres master database")
-
-		var dbErr error
-		_db_connection, dbErr := sql.Open("postgres", sqlUrl)
-		if dbErr != nil {
-			_db_connection.Close()
-			_db_connection = nil
-			panic(dbErr)
-		}
-
-		pingErr := _db_connection.Ping()
-		if pingErr != nil {
-			_db_connection.Close()
-			_db_connection = nil
-			panic(pingErr)
-		}
-
-		glog.V(3).Infoln("Checking to see if DB exists: ", conf.DataBase)
-		var existsErr error
-		dbExists, existsErr := databaseExists(_db_connection, conf.DataBase)
-		if existsErr != nil {
-			_db_connection.Close()
-			_db_connection = nil
-			panic(existsErr)
-		}
-
-		if !dbExists {
-			glog.V(3).Infoln("Database doesn't exist. Attempting to create one:  ", conf.DataBase)
-			createErr := createDatabase(_db_connection, conf.DataBase)
-			if createErr != nil {
-				_db_connection.Close()
-				_db_connection = nil
-				panic(createErr)
-			}
-		}
-
-		glog.V(3).Infoln("Closing master postgres database and opening configured database: ", conf.DataBase)
-		_db_connection.Close()
-		_db_connection = nil
-
-		sqlUrl = fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30", conf.HostName, conf.Port, conf.User, conf.Password, conf.DataBase, conf.SslMode)
-		_db_connection, dbErr = sql.Open("postgres", sqlUrl)
-		if dbErr != nil {
-			_db_connection.Close()
-			_db_connection = nil
-			panic(dbErr)
-		}
-
-		pingErr = _db_connection.Ping()
-		if pingErr != nil {
-			_db_connection.Close()
-			_db_connection = nil
-			panic(pingErr)
-		}
-
-		var maxIdleConnections, maxOpenConnections int
-
-		if conf.MaxIdleConnections != 0 {
-			maxIdleConnections = conf.MaxIdleConnections
-		} else {
-			maxIdleConnections = default_maxIdleConnections
-		}
-		if conf.MaxOpenConnections != 0 {
-			maxOpenConnections = conf.MaxOpenConnections
-		} else {
-			maxOpenConnections = default_maxOpenConnections
-		}
-
-		_db_connection.SetMaxIdleConns(maxIdleConnections)
-		_db_connection.SetMaxOpenConns(maxOpenConnections)
-	})
-	return _db_connection
-}
-
-//func NewPostgresStore(master string, confs []PostgresConf, isSharding bool, shardCount int) *PostgresStore {
-func NewPostgresStore(master string, conf PostgresConf) *PostgresStore {
-	pg := &PostgresStore{
-		db: getDbConnection(conf),
-	}
-
-	pg.createDirectoriesTable()
-
-	if err := pg.createFilesTable(); err != nil {
-		fmt.Printf("create table failed %v", err)
-	}
-
-	return pg
-}
-
-func (s *PostgresStore) Get(fullFilePath string) (fid string, err error) {
-	if err != nil {
-		return "", fmt.Errorf("PostgresStore Get operation can not parse file path %s: err is %v", fullFilePath, err)
-	}
-	fid, err = s.query(fullFilePath)
-
-	return fid, err
-}
-
-func (s *PostgresStore) Put(fullFilePath string, fid string) (err error) {
-	var old_fid string
-	if old_fid, err = s.query(fullFilePath); err != nil && err != sql.ErrNoRows {
-		return fmt.Errorf("PostgresStore Put operation failed when querying path %s: err is %v", fullFilePath, err)
-	} else {
-		if len(old_fid) == 0 {
-			err = s.insert(fullFilePath, fid)
-			if err != nil {
-				return fmt.Errorf("PostgresStore Put operation failed when inserting path %s with fid %s : err is %v", fullFilePath, fid, err)
-			}
-		} else {
-			err = s.update(fullFilePath, fid)
-			if err != nil {
-				return fmt.Errorf("PostgresStore Put operation failed when updating path %s with fid %s : err is %v", fullFilePath, fid, err)
-			}
-		}
-	}
-	return
-}
-
-func (s *PostgresStore) Delete(fullFilePath string) (err error) {
-	var fid string
-	if err != nil {
-		return fmt.Errorf("PostgresStore Delete operation can not parse file path %s: err is %v", fullFilePath, err)
-	}
-	if fid, err = s.query(fullFilePath); err != nil {
-		return fmt.Errorf("PostgresStore Delete operation failed when querying path %s: err is %v", fullFilePath, err)
-	} else if fid == "" {
-		return nil
-	}
-	if err = s.delete(fullFilePath); err != nil {
-		return fmt.Errorf("PostgresStore Delete operation failed when deleting path %s: err is %v", fullFilePath, err)
-	} else {
-		return nil
-	}
-}
-
-func (s *PostgresStore) Close() {
-	s.db.Close()
-}
-
-func (s *PostgresStore) FindDir(dirPath string) (dirId filer.DirectoryId, err error) {
-	dirId, _, err = s.lookupDirectory(dirPath)
-	return dirId, err
-}
-
-func (s *PostgresStore) ListDirs(dirPath string) (dirs []filer.DirectoryEntry, err error) {
-	dirs, err = s.findDirectories(dirPath, 20)
-
-	glog.V(3).Infof("Postgres ListDirs = found %d directories under %s", len(dirs), dirPath)
-
-	return dirs, err
-}
-
-func (s *PostgresStore) DeleteDir(dirPath string, recursive bool) (err error) {
-	err = s.deleteDirectory(dirPath, recursive)
-	if err != nil {
-		glog.V(0).Infof("Error in Postgres DeleteDir '%s' (recursive = '%t'): %s", err)
-	}
-	return err
-}
-
-func (s *PostgresStore) FindFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
-	files, err = s.findFiles(dirPath, lastFileName, limit)
-
-	return files, err
-}
-
-var createDirectoryTable = `
-
-CREATE TABLE IF NOT EXISTS %s (
-  id BIGSERIAL NOT NULL,
-  directoryRoot VARCHAR(1024) NOT NULL DEFAULT '',
-  directoryName VARCHAR(1024) NOT NULL DEFAULT '',
-  CONSTRAINT unique_directory UNIQUE (directoryRoot, directoryName)
-);
-`
-
-var createFileTable = `
-
-CREATE TABLE IF NOT EXISTS %s (
-  id BIGSERIAL NOT NULL,
-  directoryPart VARCHAR(1024) NOT NULL DEFAULT '',
-  filePart VARCHAR(1024) NOT NULL DEFAULT '',
-  fid VARCHAR(36) NOT NULL DEFAULT '',
-  createTime BIGINT NOT NULL DEFAULT 0,
-  updateTime BIGINT NOT NULL DEFAULT 0,
-  remark VARCHAR(20) NOT NULL DEFAULT '',
-  status SMALLINT NOT NULL DEFAULT '1',
-  PRIMARY KEY (id),
-  CONSTRAINT %s_unique_file UNIQUE (directoryPart, filePart)
-);
-`
-
-func (s *PostgresStore) createDirectoriesTable() error {
-	glog.V(3).Infoln("Creating postgres table if it doesn't exist: ", directoriesTableName)
-
-	sqlCreate := fmt.Sprintf(createDirectoryTable, directoriesTableName)
-
-	stmt, err := s.db.Prepare(sqlCreate)
-	if err != nil {
-		return err
-	}
-	defer stmt.Close()
-
-	_, err = stmt.Exec()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *PostgresStore) createFilesTable() error {
-
-	glog.V(3).Infoln("Creating postgres table if it doesn't exist: ", filesTableName)
-
-	sqlCreate := fmt.Sprintf(createFileTable, filesTableName, filesTableName)
-
-	stmt, err := s.db.Prepare(sqlCreate)
-	if err != nil {
-		return err
-	}
-	defer stmt.Close()
-
-	_, err = stmt.Exec()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *PostgresStore) query(uriPath string) (string, error) {
-	directoryPart, filePart := filepath.Split(uriPath)
-	sqlStatement := fmt.Sprintf("SELECT fid FROM %s WHERE directoryPart=$1 AND filePart=$2", filesTableName)
-
-	row := s.db.QueryRow(sqlStatement, directoryPart, filePart)
-	var fid string
-	err := row.Scan(&fid)
-
-	glog.V(3).Infof("Postgres query -- looking up path '%s' and found id '%s' ", uriPath, fid)
-
-	if err != nil {
-		return "", err
-	}
-	return fid, nil
-}
-
-func (s *PostgresStore) update(uriPath string, fid string) error {
-	directoryPart, filePart := filepath.Split(uriPath)
-	sqlStatement := fmt.Sprintf("UPDATE %s SET fid=$1, updateTime=$2 WHERE directoryPart=$3 AND filePart=$4", filesTableName)
-
-	glog.V(3).Infof("Postgres query -- updating path '%s' with id '%s'", uriPath, fid)
-
-	res, err := s.db.Exec(sqlStatement, fid, time.Now().Unix(), directoryPart, filePart)
-	if err != nil {
-		return err
-	}
-
-	_, err = res.RowsAffected()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *PostgresStore) insert(uriPath string, fid string) error {
-	directoryPart, filePart := filepath.Split(uriPath)
-
-	existingId, _, _ := s.lookupDirectory(directoryPart)
-	if existingId == 0 {
-		s.recursiveInsertDirectory(directoryPart)
-	}
-
-	sqlStatement := fmt.Sprintf("INSERT INTO %s (directoryPart,filePart,fid,createTime) VALUES($1, $2, $3, $4)", filesTableName)
-	glog.V(3).Infof("Postgres query -- inserting path '%s' with id '%s'", uriPath, fid)
-
-	res, err := s.db.Exec(sqlStatement, directoryPart, filePart, fid, time.Now().Unix())
-
-	if err != nil {
-		return err
-	}
-
-	rows, err := res.RowsAffected()
-	if rows != 1 {
-		return fmt.Errorf("Postgres insert -- rows affected = %d. Expecting 1", rows)
-	}
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (s *PostgresStore) recursiveInsertDirectory(dirPath string) {
-	pathParts := strings.Split(dirPath, "/")
-
-	var workingPath string = "/"
-	for _, part := range pathParts {
-		if part == "" {
-			continue
-		}
-		workingPath += (part + "/")
-		existingId, _, _ := s.lookupDirectory(workingPath)
-		if existingId == 0 {
-			s.insertDirectory(workingPath)
-		}
-	}
-}
-
-func (s *PostgresStore) insertDirectory(dirPath string) {
-	pathParts := strings.Split(dirPath, "/")
-
-	directoryRoot := "/"
-	directoryName := ""
-	if len(pathParts) > 1 {
-		directoryRoot = strings.Join(pathParts[0:len(pathParts)-2], "/") + "/"
-		directoryName = strings.Join(pathParts[len(pathParts)-2:], "/")
-	} else if len(pathParts) == 1 {
-		directoryRoot = "/"
-		directoryName = pathParts[0] + "/"
-	}
-	sqlInsertDirectoryStatement := fmt.Sprintf("INSERT INTO %s (directoryroot, directoryname)  "+
-		"SELECT $1, $2 WHERE NOT EXISTS ( SELECT id FROM %s WHERE directoryroot=$3 AND directoryname=$4 )",
-		directoriesTableName, directoriesTableName)
-
-	glog.V(4).Infof("Postgres query -- Inserting directory (if it doesn't exist) - root = %s, name = %s",
-		directoryRoot, directoryName)
-
-	_, err := s.db.Exec(sqlInsertDirectoryStatement, directoryRoot, directoryName, directoryRoot, directoryName)
-	if err != nil {
-		glog.V(0).Infof("Postgres query -- Error inserting directory - root = %s, name = %s: %s",
-			directoryRoot, directoryName, err)
-	}
-}
-
-func (s *PostgresStore) delete(uriPath string) error {
-	directoryPart, filePart := filepath.Split(uriPath)
-	sqlStatement := fmt.Sprintf("DELETE FROM %s WHERE directoryPart=$1 AND filePart=$2", filesTableName)
-
-	glog.V(3).Infof("Postgres query -- deleting path '%s'", uriPath)
-
-	res, err := s.db.Exec(sqlStatement, directoryPart, filePart)
-	if err != nil {
-		return err
-	}
-
-	_, err = res.RowsAffected()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *PostgresStore) lookupDirectory(dirPath string) (filer.DirectoryId, string, error) {
-	directoryRoot, directoryName := s.mySplitPath(dirPath)
-
-	sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName)
-
-	row := s.db.QueryRow(sqlStatement, directoryRoot, directoryName)
-	var id filer.DirectoryId
-	var dirRoot string
-	var dirName string
-	err := row.Scan(&id, &dirRoot, &dirName)
-
-	glog.V(3).Infof("Postgres lookupDirectory -- looking up directory '%s' and found id '%d', root '%s', name '%s' ", dirPath, id, dirRoot, dirName)
-
-	if err != nil {
-		return 0, "", err
-	}
-	return id, filepath.Join(dirRoot, dirName), err
-}
-
-func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer.DirectoryEntry, err error) {
-	sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName != '' ORDER BY id LIMIT $2", directoriesTableName)
-	rows, err := s.db.Query(sqlStatement, dirPath, limit)
-
-	if err != nil {
-		glog.V(0).Infof("Postgres findDirectories error: %s", err)
-	}
-
-	if rows != nil {
-		defer rows.Close()
-		for rows.Next() {
-			var id filer.DirectoryId
-			var directoryRoot string
-			var directoryName string
-
-			scanErr := rows.Scan(&id, &directoryRoot, &directoryName)
-			if scanErr != nil {
-				err = scanErr
-			}
-			dirs = append(dirs, filer.DirectoryEntry{Name: (directoryName), Id: id})
-		}
-	}
-	return
-}
-
-func (s *PostgresStore) safeToDeleteDirectory(dirPath string, recursive bool) bool {
-	if recursive {
-		return true
-	}
-	sqlStatement := fmt.Sprintf("SELECT id FROM %s WHERE directoryRoot LIKE $1 LIMIT 1", directoriesTableName)
-	row := s.db.QueryRow(sqlStatement, dirPath+"%")
-
-	var id filer.DirectoryId
-	err := row.Scan(&id)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return true
-		}
-	}
-	return false
-}
-
-func (s *PostgresStore) mySplitPath(dirPath string) (directoryRoot string, directoryName string) {
-	pathParts := strings.Split(dirPath, "/")
-	directoryRoot = "/"
-	directoryName = ""
-	if len(pathParts) > 1 {
-		directoryRoot = strings.Join(pathParts[0:len(pathParts)-2], "/") + "/"
-		directoryName = strings.Join(pathParts[len(pathParts)-2:], "/")
-	} else if len(pathParts) == 1 {
-		directoryRoot = "/"
-		directoryName = pathParts[0] + "/"
-	}
-	return directoryRoot, directoryName
-}
-
-func (s *PostgresStore) deleteDirectory(dirPath string, recursive bool) (err error) {
-	directoryRoot, directoryName := s.mySplitPath(dirPath)
-
-	// delete files
-	sqlStatement := fmt.Sprintf("DELETE FROM %s WHERE directorypart=$1", filesTableName)
-	_, err = s.db.Exec(sqlStatement, dirPath)
-	if err != nil {
-		return err
-	}
-
-	// delete specific directory if it is empty or recursive delete was requested
-	safeToDelete := s.safeToDeleteDirectory(dirPath, recursive)
-	if safeToDelete {
-		sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName)
-		_, err = s.db.Exec(sqlStatement, directoryRoot, directoryName)
-		if err != nil {
-			return err
-		}
-	}
-
-	if recursive {
-		// delete descendant files
-		sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directorypart LIKE $1", filesTableName)
-		_, err = s.db.Exec(sqlStatement, dirPath+"%")
-		if err != nil {
-			return err
-		}
-
-		// delete descendant directories
-		sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directoryRoot LIKE $1", directoriesTableName)
-		_, err = s.db.Exec(sqlStatement, dirPath+"%")
-		if err != nil {
-			return err
-		}
-	}
-
-	return err
-}
-
-func (s *PostgresStore) findFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
-	var rows *sql.Rows = nil
-
-	if lastFileName == "" {
-		sqlStatement :=
-			fmt.Sprintf("SELECT fid, directorypart, filepart FROM %s WHERE directorypart=$1 ORDER BY id LIMIT $2", filesTableName)
-		rows, err = s.db.Query(sqlStatement, dirPath, limit)
-	} else {
-		sqlStatement :=
-			fmt.Sprintf("SELECT fid, directorypart, filepart FROM %s WHERE directorypart=$1 "+
-				"AND id > (SELECT id FROM %s WHERE directoryPart=$2 AND filepart=$3)  ORDER BY id LIMIT $4",
-				filesTableName, filesTableName)
-		_, lastFileNameName := filepath.Split(lastFileName)
-		rows, err = s.db.Query(sqlStatement, dirPath, dirPath, lastFileNameName, limit)
-	}
-
-	if err != nil {
-		glog.V(0).Infof("Postgres find files error: %s", err)
-	}
-
-	if rows != nil {
-		defer rows.Close()
-
-		for rows.Next() {
-			var fid filer.FileId
-			var directoryPart string
-			var filePart string
-
-			scanErr := rows.Scan(&fid, &directoryPart, &filePart)
-			if scanErr != nil {
-				err = scanErr
-			}
-
-			files = append(files, filer.FileEntry{Name: filepath.Join(directoryPart, filePart), Id: fid})
-			if len(files) >= limit {
-				break
-			}
-		}
-	}
-
-	glog.V(3).Infof("Postgres findFiles -- looking up files under '%s' and found %d files. Limit=%d, lastFileName=%s",
-		dirPath, len(files), limit, lastFileName)
-
-	return files, err
-}

weed/filer/redis_store/redis_store.go

@@ -1,50 +0,0 @@
-package redis_store
-
-import (
-	"github.com/chrislusf/seaweedfs/weed/filer"
-
-	redis "gopkg.in/redis.v2"
-)
-
-type RedisStore struct {
-	Client *redis.Client
-}
-
-func NewRedisStore(hostPort string, password string, database int) *RedisStore {
-	client := redis.NewTCPClient(&redis.Options{
-		Addr:     hostPort,
-		Password: password,
-		DB:       int64(database),
-	})
-	return &RedisStore{Client: client}
-}
-
-func (s *RedisStore) Get(fullFileName string) (fid string, err error) {
-	fid, err = s.Client.Get(fullFileName).Result()
-	if err == redis.Nil {
-		err = filer.ErrNotFound
-	}
-	return fid, err
-}
-func (s *RedisStore) Put(fullFileName string, fid string) (err error) {
-	_, err = s.Client.Set(fullFileName, fid).Result()
-	if err == redis.Nil {
-		err = nil
-	}
-	return err
-}
-
-// Currently the fid is not returned
-func (s *RedisStore) Delete(fullFileName string) (err error) {
-	_, err = s.Client.Del(fullFileName).Result()
-	if err == redis.Nil {
-		err = nil
-	}
-	return err
-}
-
-func (s *RedisStore) Close() {
-	if s.Client != nil {
-		s.Client.Close()
-	}
-}

weed/filer2/abstract_sql/abstract_sql_store.go

@@ -0,0 +1,168 @@
+package abstract_sql
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+)
+
+type AbstractSqlStore struct {
+	DB               *sql.DB
+	SqlInsert        string
+	SqlUpdate        string
+	SqlFind          string
+	SqlDelete        string
+	SqlListExclusive string
+	SqlListInclusive string
+}
+
+type TxOrDB interface {
+	ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
+	QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row
+	QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
+}
+
+func (store *AbstractSqlStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	tx, err := store.DB.BeginTx(ctx, &sql.TxOptions{
+		Isolation: sql.LevelReadCommitted,
+		ReadOnly:  false,
+	})
+	if err != nil {
+		return ctx, err
+	}
+
+	return context.WithValue(ctx, "tx", tx), nil
+}
+func (store *AbstractSqlStore) CommitTransaction(ctx context.Context) error {
+	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
+		return tx.Commit()
+	}
+	return nil
+}
+func (store *AbstractSqlStore) RollbackTransaction(ctx context.Context) error {
+	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
+		return tx.Rollback()
+	}
+	return nil
+}
+
+func (store *AbstractSqlStore) getTxOrDB(ctx context.Context) TxOrDB {
+	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
+		return tx
+	}
+	return store.DB
+}
+
+func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+
+	dir, name := entry.FullPath.DirAndName()
+	meta, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
+	}
+
+	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, hashToLong(dir), name, dir, meta)
+	if err != nil {
+		return fmt.Errorf("insert %s: %s", entry.FullPath, err)
+	}
+
+	_, err = res.RowsAffected()
+	if err != nil {
+		return fmt.Errorf("insert %s but no rows affected: %s", entry.FullPath, err)
+	}
+	return nil
+}
+
+func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+
+	dir, name := entry.FullPath.DirAndName()
+	meta, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
+	}
+
+	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, hashToLong(dir), name, dir)
+	if err != nil {
+		return fmt.Errorf("update %s: %s", entry.FullPath, err)
+	}
+
+	_, err = res.RowsAffected()
+	if err != nil {
+		return fmt.Errorf("update %s but no rows affected: %s", entry.FullPath, err)
+	}
+	return nil
+}
+
+func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (*filer2.Entry, error) {
+
+	dir, name := fullpath.DirAndName()
+	row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, hashToLong(dir), name, dir)
+	var data []byte
+	if err := row.Scan(&data); err != nil {
+		return nil, filer2.ErrNotFound
+	}
+
+	entry := &filer2.Entry{
+		FullPath: fullpath,
+	}
+	if err := entry.DecodeAttributesAndChunks(data); err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) error {
+
+	dir, name := fullpath.DirAndName()
+
+	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, hashToLong(dir), name, dir)
+	if err != nil {
+		return fmt.Errorf("delete %s: %s", fullpath, err)
+	}
+
+	_, err = res.RowsAffected()
+	if err != nil {
+		return fmt.Errorf("delete %s but no rows affected: %s", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
+
+	sqlText := store.SqlListExclusive
+	if inclusive {
+		sqlText = store.SqlListInclusive
+	}
+
+	rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, hashToLong(string(fullpath)), startFileName, string(fullpath), limit)
+	if err != nil {
+		return nil, fmt.Errorf("list %s : %v", fullpath, err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var name string
+		var data []byte
+		if err = rows.Scan(&name, &data); err != nil {
+			glog.V(0).Infof("scan %s : %v", fullpath, err)
+			return nil, fmt.Errorf("scan %s: %v", fullpath, err)
+		}
+
+		entry := &filer2.Entry{
+			FullPath: filer2.NewFullPath(string(fullpath), name),
+		}
+		if err = entry.DecodeAttributesAndChunks(data); err != nil {
+			glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err)
+			return nil, fmt.Errorf("scan decode %s : %v", entry.FullPath, err)
+		}
+
+		entries = append(entries, entry)
+	}
+
+	return entries, nil
+}

weed/filer2/abstract_sql/hashing.go

@@ -0,0 +1,32 @@
+package abstract_sql
+
+import (
+	"crypto/md5"
+	"io"
+)
+
+// returns a 64 bit big int
+func hashToLong(dir string) (v int64) {
+	h := md5.New()
+	io.WriteString(h, dir)
+
+	b := h.Sum(nil)
+
+	v += int64(b[0])
+	v <<= 8
+	v += int64(b[1])
+	v <<= 8
+	v += int64(b[2])
+	v <<= 8
+	v += int64(b[3])
+	v <<= 8
+	v += int64(b[4])
+	v <<= 8
+	v += int64(b[5])
+	v <<= 8
+	v += int64(b[6])
+	v <<= 8
+	v += int64(b[7])
+
+	return
+}

weed/filer2/cassandra/README.txt

@@ -0,0 +1,14 @@
+1. create a keyspace
+
+CREATE KEYSPACE seaweedfs WITH replication = {'class':'SimpleStrategy', 'replication_factor' : 1};
+
+2. create filemeta table
+
+ USE seaweedfs;
+
+ CREATE TABLE filemeta (
+    directory varchar,
+    name varchar,
+    meta blob,
+    PRIMARY KEY (directory, name)
+ ) WITH CLUSTERING ORDER BY (name ASC);

weed/filer2/cassandra/cassandra_store.go

@@ -0,0 +1,142 @@
+package cassandra
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/gocql/gocql"
+)
+
+func init() {
+	filer2.Stores = append(filer2.Stores, &CassandraStore{})
+}
+
+type CassandraStore struct {
+	cluster *gocql.ClusterConfig
+	session *gocql.Session
+}
+
+func (store *CassandraStore) GetName() string {
+	return "cassandra"
+}
+
+func (store *CassandraStore) Initialize(configuration util.Configuration) (err error) {
+	return store.initialize(
+		configuration.GetString("keyspace"),
+		configuration.GetStringSlice("hosts"),
+	)
+}
+
+func (store *CassandraStore) initialize(keyspace string, hosts []string) (err error) {
+	store.cluster = gocql.NewCluster(hosts...)
+	store.cluster.Keyspace = keyspace
+	store.cluster.Consistency = gocql.LocalQuorum
+	store.session, err = store.cluster.CreateSession()
+	if err != nil {
+		glog.V(0).Infof("Failed to open cassandra store, hosts %v, keyspace %s", hosts, keyspace)
+	}
+	return
+}
+
+func (store *CassandraStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *CassandraStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *CassandraStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *CassandraStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+
+	dir, name := entry.FullPath.DirAndName()
+	meta, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
+	}
+
+	if err := store.session.Query(
+		"INSERT INTO filemeta (directory,name,meta) VALUES(?,?,?) USING TTL ? ",
+		dir, name, meta, entry.TtlSec).Exec(); err != nil {
+		return fmt.Errorf("insert %s: %s", entry.FullPath, err)
+	}
+
+	return nil
+}
+
+func (store *CassandraStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *CassandraStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+
+	dir, name := fullpath.DirAndName()
+	var data []byte
+	if err := store.session.Query(
+		"SELECT meta FROM filemeta WHERE directory=? AND name=?",
+		dir, name).Consistency(gocql.One).Scan(&data); err != nil {
+		if err != gocql.ErrNotFound {
+			return nil, filer2.ErrNotFound
+		}
+	}
+
+	if len(data) == 0 {
+		return nil, fmt.Errorf("not found: %s", fullpath)
+	}
+
+	entry = &filer2.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(data)
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *CassandraStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) error {
+
+	dir, name := fullpath.DirAndName()
+
+	if err := store.session.Query(
+		"DELETE FROM filemeta WHERE directory=? AND name=?",
+		dir, name).Exec(); err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
+	limit int) (entries []*filer2.Entry, err error) {
+
+	cqlStr := "SELECT NAME, meta FROM filemeta WHERE directory=? AND name>? ORDER BY NAME ASC LIMIT ?"
+	if inclusive {
+		cqlStr = "SELECT NAME, meta FROM filemeta WHERE directory=? AND name>=? ORDER BY NAME ASC LIMIT ?"
+	}
+
+	var data []byte
+	var name string
+	iter := store.session.Query(cqlStr, string(fullpath), startFileName, limit).Iter()
+	for iter.Scan(&name, &data) {
+		entry := &filer2.Entry{
+			FullPath: filer2.NewFullPath(string(fullpath), name),
+		}
+		if decodeErr := entry.DecodeAttributesAndChunks(data); decodeErr != nil {
+			err = decodeErr
+			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
+			break
+		}
+		entries = append(entries, entry)
+	}
+	if err := iter.Close(); err != nil {
+		glog.V(0).Infof("list iterator close: %v", err)
+	}
+
+	return entries, err
+}

weed/filer2/configuration.go

@@ -0,0 +1,51 @@
+package filer2
+
+import (
+	"os"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/spf13/viper"
+)
+
+var (
+	Stores []FilerStore
+)
+
+func (f *Filer) LoadConfiguration(config *viper.Viper) {
+
+	validateOneEnabledStore(config)
+
+	for _, store := range Stores {
+		if config.GetBool(store.GetName() + ".enabled") {
+			viperSub := config.Sub(store.GetName())
+			if err := store.Initialize(viperSub); err != nil {
+				glog.Fatalf("Failed to initialize store for %s: %+v",
+					store.GetName(), err)
+			}
+			f.SetStore(store)
+			glog.V(0).Infof("Configure filer for %s", store.GetName())
+			return
+		}
+	}
+
+	println()
+	println("Supported filer stores are:")
+	for _, store := range Stores {
+		println("    " + store.GetName())
+	}
+
+	os.Exit(-1)
+}
+
+func validateOneEnabledStore(config *viper.Viper) {
+	enabledStore := ""
+	for _, store := range Stores {
+		if config.GetBool(store.GetName() + ".enabled") {
+			if enabledStore == "" {
+				enabledStore = store.GetName()
+			} else {
+				glog.Fatalf("Filer store is enabled for both %s and %s", enabledStore, store.GetName())
+			}
+		}
+	}
+}

weed/filer2/entry.go

@@ -0,0 +1,71 @@
+package filer2
+
+import (
+	"os"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+)
+
+type Attr struct {
+	Mtime         time.Time   // time of last modification
+	Crtime        time.Time   // time of creation (OS X only)
+	Mode          os.FileMode // file mode
+	Uid           uint32      // owner uid
+	Gid           uint32      // group gid
+	Mime          string      // mime type
+	Replication   string      // replication
+	Collection    string      // collection name
+	TtlSec        int32       // ttl in seconds
+	UserName      string
+	GroupNames    []string
+	SymlinkTarget string
+}
+
+func (attr Attr) IsDirectory() bool {
+	return attr.Mode&os.ModeDir > 0
+}
+
+type Entry struct {
+	FullPath
+
+	Attr
+
+	// the following is for files
+	Chunks []*filer_pb.FileChunk `json:"chunks,omitempty"`
+}
+
+func (entry *Entry) Size() uint64 {
+	return TotalSize(entry.Chunks)
+}
+
+func (entry *Entry) Timestamp() time.Time {
+	if entry.IsDirectory() {
+		return entry.Crtime
+	} else {
+		return entry.Mtime
+	}
+}
+
+func (entry *Entry) ToProtoEntry() *filer_pb.Entry {
+	if entry == nil {
+		return nil
+	}
+	return &filer_pb.Entry{
+		Name:        entry.FullPath.Name(),
+		IsDirectory: entry.IsDirectory(),
+		Attributes:  EntryAttributeToPb(entry),
+		Chunks:      entry.Chunks,
+	}
+}
+
+func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry {
+	if entry == nil {
+		return nil
+	}
+	dir, _ := entry.FullPath.DirAndName()
+	return &filer_pb.FullEntry{
+		Dir:   dir,
+		Entry: entry.ToProtoEntry(),
+	}
+}

weed/filer2/entry_codec.go

@@ -0,0 +1,93 @@
+package filer2
+
+import (
+	"os"
+	"time"
+
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/golang/protobuf/proto"
+)
+
+func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {
+	message := &filer_pb.Entry{
+		Attributes: EntryAttributeToPb(entry),
+		Chunks:     entry.Chunks,
+	}
+	return proto.Marshal(message)
+}
+
+func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error {
+
+	message := &filer_pb.Entry{}
+
+	if err := proto.UnmarshalMerge(blob, message); err != nil {
+		return fmt.Errorf("decoding value blob for %s: %v", entry.FullPath, err)
+	}
+
+	entry.Attr = PbToEntryAttribute(message.Attributes)
+
+	entry.Chunks = message.Chunks
+
+	return nil
+}
+
+func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes {
+
+	return &filer_pb.FuseAttributes{
+		Crtime:        entry.Attr.Crtime.Unix(),
+		Mtime:         entry.Attr.Mtime.Unix(),
+		FileMode:      uint32(entry.Attr.Mode),
+		Uid:           entry.Uid,
+		Gid:           entry.Gid,
+		Mime:          entry.Mime,
+		Collection:    entry.Attr.Collection,
+		Replication:   entry.Attr.Replication,
+		TtlSec:        entry.Attr.TtlSec,
+		UserName:      entry.Attr.UserName,
+		GroupName:     entry.Attr.GroupNames,
+		SymlinkTarget: entry.Attr.SymlinkTarget,
+	}
+}
+
+func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr {
+
+	t := Attr{}
+
+	t.Crtime = time.Unix(attr.Crtime, 0)
+	t.Mtime = time.Unix(attr.Mtime, 0)
+	t.Mode = os.FileMode(attr.FileMode)
+	t.Uid = attr.Uid
+	t.Gid = attr.Gid
+	t.Mime = attr.Mime
+	t.Collection = attr.Collection
+	t.Replication = attr.Replication
+	t.TtlSec = attr.TtlSec
+	t.UserName = attr.UserName
+	t.GroupNames = attr.GroupName
+	t.SymlinkTarget = attr.SymlinkTarget
+
+	return t
+}
+
+func EqualEntry(a, b *Entry) bool {
+	if a == b {
+		return true
+	}
+	if a == nil && b != nil || a != nil && b == nil {
+		return false
+	}
+	if !proto.Equal(EntryAttributeToPb(a), EntryAttributeToPb(b)) {
+		return false
+	}
+	if len(a.Chunks) != len(b.Chunks) {
+		return false
+	}
+
+	for i := 0; i < len(a.Chunks); i++ {
+		if !proto.Equal(a.Chunks[i], b.Chunks[i]) {
+			return false
+		}
+	}
+	return true
+}

weed/filer2/filechunks.go

@@ -0,0 +1,228 @@
+package filer2
+
+import (
+	"fmt"
+	"hash/fnv"
+	"sort"
+	"sync"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+)
+
+func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
+	for _, c := range chunks {
+		t := uint64(c.Offset + int64(c.Size))
+		if size < t {
+			size = t
+		}
+	}
+	return
+}
+
+func ETag(chunks []*filer_pb.FileChunk) (etag string) {
+	if len(chunks) == 1 {
+		return chunks[0].ETag
+	}
+
+	h := fnv.New32a()
+	for _, c := range chunks {
+		h.Write([]byte(c.ETag))
+	}
+	return fmt.Sprintf("%x", h.Sum32())
+}
+
+func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
+
+	visibles := NonOverlappingVisibleIntervals(chunks)
+
+	fileIds := make(map[string]bool)
+	for _, interval := range visibles {
+		fileIds[interval.fileId] = true
+	}
+	for _, chunk := range chunks {
+		if found := fileIds[chunk.FileId]; found {
+			compacted = append(compacted, chunk)
+		} else {
+			garbage = append(garbage, chunk)
+		}
+	}
+
+	return
+}
+
+func FindUnusedFileChunks(oldChunks, newChunks []*filer_pb.FileChunk) (unused []*filer_pb.FileChunk) {
+
+	fileIds := make(map[string]bool)
+	for _, interval := range newChunks {
+		fileIds[interval.FileId] = true
+	}
+	for _, chunk := range oldChunks {
+		if found := fileIds[chunk.FileId]; !found {
+			unused = append(unused, chunk)
+		}
+	}
+
+	return
+}
+
+type ChunkView struct {
+	FileId      string
+	Offset      int64
+	Size        uint64
+	LogicOffset int64
+	IsFullChunk bool
+}
+
+func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) {
+
+	visibles := NonOverlappingVisibleIntervals(chunks)
+
+	return ViewFromVisibleIntervals(visibles, offset, size)
+
+}
+
+func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int) (views []*ChunkView) {
+
+	stop := offset + int64(size)
+
+	for _, chunk := range visibles {
+		if chunk.start <= offset && offset < chunk.stop && offset < stop {
+			isFullChunk := chunk.isFullChunk && chunk.start == offset && chunk.stop <= stop
+			views = append(views, &ChunkView{
+				FileId:      chunk.fileId,
+				Offset:      offset - chunk.start, // offset is the data starting location in this file id
+				Size:        uint64(min(chunk.stop, stop) - offset),
+				LogicOffset: offset,
+				IsFullChunk: isFullChunk,
+			})
+			offset = min(chunk.stop, stop)
+		}
+	}
+
+	return views
+
+}
+
+func logPrintf(name string, visibles []VisibleInterval) {
+	/*
+		log.Printf("%s len %d", name, len(visibles))
+		for _, v := range visibles {
+			log.Printf("%s:  => %+v", name, v)
+		}
+	*/
+}
+
+var bufPool = sync.Pool{
+	New: func() interface{} {
+		return new(VisibleInterval)
+	},
+}
+
+func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.FileChunk) []VisibleInterval {
+
+	newV := newVisibleInterval(
+		chunk.Offset,
+		chunk.Offset+int64(chunk.Size),
+		chunk.FileId,
+		chunk.Mtime,
+		true,
+	)
+
+	length := len(visibles)
+	if length == 0 {
+		return append(visibles, newV)
+	}
+	last := visibles[length-1]
+	if last.stop <= chunk.Offset {
+		return append(visibles, newV)
+	}
+
+	logPrintf("  before", visibles)
+	for _, v := range visibles {
+		if v.start < chunk.Offset && chunk.Offset < v.stop {
+			newVisibles = append(newVisibles, newVisibleInterval(
+				v.start,
+				chunk.Offset,
+				v.fileId,
+				v.modifiedTime,
+				false,
+			))
+		}
+		chunkStop := chunk.Offset + int64(chunk.Size)
+		if v.start < chunkStop && chunkStop < v.stop {
+			newVisibles = append(newVisibles, newVisibleInterval(
+				chunkStop,
+				v.stop,
+				v.fileId,
+				v.modifiedTime,
+				false,
+			))
+		}
+		if chunkStop <= v.start || v.stop <= chunk.Offset {
+			newVisibles = append(newVisibles, v)
+		}
+	}
+	newVisibles = append(newVisibles, newV)
+
+	logPrintf("  append", newVisibles)
+
+	for i := len(newVisibles) - 1; i >= 0; i-- {
+		if i > 0 && newV.start < newVisibles[i-1].start {
+			newVisibles[i] = newVisibles[i-1]
+		} else {
+			newVisibles[i] = newV
+			break
+		}
+	}
+	logPrintf("  sorted", newVisibles)
+
+	return newVisibles
+}
+
+func NonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) {
+
+	sort.Slice(chunks, func(i, j int) bool {
+		return chunks[i].Mtime < chunks[j].Mtime
+	})
+
+	var newVisibles []VisibleInterval
+	for _, chunk := range chunks {
+		newVisibles = MergeIntoVisibles(visibles, newVisibles, chunk)
+		t := visibles[:0]
+		visibles = newVisibles
+		newVisibles = t
+
+		logPrintf("add", visibles)
+
+	}
+
+	return
+}
+
+// find non-overlapping visible intervals
+// visible interval map to one file chunk
+
+type VisibleInterval struct {
+	start        int64
+	stop         int64
+	modifiedTime int64
+	fileId       string
+	isFullChunk  bool
+}
+
+func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool) VisibleInterval {
+	return VisibleInterval{
+		start:        start,
+		stop:         stop,
+		fileId:       fileId,
+		modifiedTime: modifiedTime,
+		isFullChunk:  isFullChunk,
+	}
+}
+
+func min(x, y int64) int64 {
+	if x <= y {
+		return x
+	}
+	return y
+}

weed/filer2/filechunks_test.go

@@ -0,0 +1,384 @@
+package filer2
+
+import (
+	"log"
+	"testing"
+
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestCompactFileChunks(t *testing.T) {
+	chunks := []*filer_pb.FileChunk{
+		{Offset: 10, Size: 100, FileId: "abc", Mtime: 50},
+		{Offset: 100, Size: 100, FileId: "def", Mtime: 100},
+		{Offset: 200, Size: 100, FileId: "ghi", Mtime: 200},
+		{Offset: 110, Size: 200, FileId: "jkl", Mtime: 300},
+	}
+
+	compacted, garbage := CompactFileChunks(chunks)
+
+	if len(compacted) != 3 {
+		t.Fatalf("unexpected compacted: %d", len(compacted))
+	}
+	if len(garbage) != 1 {
+		t.Fatalf("unexpected garbage: %d", len(garbage))
+	}
+
+}
+
+func TestCompactFileChunks2(t *testing.T) {
+
+	chunks := []*filer_pb.FileChunk{
+		{Offset: 0, Size: 100, FileId: "abc", Mtime: 50},
+		{Offset: 100, Size: 100, FileId: "def", Mtime: 100},
+		{Offset: 200, Size: 100, FileId: "ghi", Mtime: 200},
+		{Offset: 0, Size: 100, FileId: "abcf", Mtime: 300},
+		{Offset: 50, Size: 100, FileId: "fhfh", Mtime: 400},
+		{Offset: 100, Size: 100, FileId: "yuyu", Mtime: 500},
+	}
+
+	k := 3
+
+	for n := 0; n < k; n++ {
+		chunks = append(chunks, &filer_pb.FileChunk{
+			Offset: int64(n * 100), Size: 100, FileId: fmt.Sprintf("fileId%d", n), Mtime: int64(n),
+		})
+		chunks = append(chunks, &filer_pb.FileChunk{
+			Offset: int64(n * 50), Size: 100, FileId: fmt.Sprintf("fileId%d", n+k), Mtime: int64(n + k),
+		})
+	}
+
+	compacted, garbage := CompactFileChunks(chunks)
+
+	if len(compacted) != 4 {
+		t.Fatalf("unexpected compacted: %d", len(compacted))
+	}
+	if len(garbage) != 8 {
+		t.Fatalf("unexpected garbage: %d", len(garbage))
+	}
+}
+
+func TestIntervalMerging(t *testing.T) {
+
+	testcases := []struct {
+		Chunks   []*filer_pb.FileChunk
+		Expected []*VisibleInterval
+	}{
+		// case 0: normal
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 100, Size: 100, FileId: "asdf", Mtime: 134},
+				{Offset: 200, Size: 100, FileId: "fsad", Mtime: 353},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 100, fileId: "abc"},
+				{start: 100, stop: 200, fileId: "asdf"},
+				{start: 200, stop: 300, fileId: "fsad"},
+			},
+		},
+		// case 1: updates overwrite full chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 200, fileId: "asdf"},
+			},
+		},
+		// case 2: updates overwrite part of previous chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 50, FileId: "asdf", Mtime: 134},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 50, fileId: "asdf"},
+				{start: 50, stop: 100, fileId: "abc"},
+			},
+		},
+		// case 3: updates overwrite full chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
+				{Offset: 50, Size: 250, FileId: "xxxx", Mtime: 154},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 50, fileId: "asdf"},
+				{start: 50, stop: 300, fileId: "xxxx"},
+			},
+		},
+		// case 4: updates far away from prev chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
+				{Offset: 250, Size: 250, FileId: "xxxx", Mtime: 154},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 200, fileId: "asdf"},
+				{start: 250, stop: 500, fileId: "xxxx"},
+			},
+		},
+		// case 5: updates overwrite full chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 184},
+				{Offset: 70, Size: 150, FileId: "abc", Mtime: 143},
+				{Offset: 80, Size: 100, FileId: "xxxx", Mtime: 134},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 200, fileId: "asdf"},
+				{start: 200, stop: 220, fileId: "abc"},
+			},
+		},
+		// case 6: same updates
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 100, fileId: "abc"},
+			},
+		},
+		// case 7: real updates
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 2097152, FileId: "7,0294cbb9892b", Mtime: 123},
+				{Offset: 0, Size: 3145728, FileId: "3,029565bf3092", Mtime: 130},
+				{Offset: 2097152, Size: 3145728, FileId: "6,029632f47ae2", Mtime: 140},
+				{Offset: 5242880, Size: 3145728, FileId: "2,029734c5aa10", Mtime: 150},
+				{Offset: 8388608, Size: 3145728, FileId: "5,02982f80de50", Mtime: 160},
+				{Offset: 11534336, Size: 2842193, FileId: "7,0299ad723803", Mtime: 170},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 2097152, fileId: "3,029565bf3092"},
+				{start: 2097152, stop: 5242880, fileId: "6,029632f47ae2"},
+				{start: 5242880, stop: 8388608, fileId: "2,029734c5aa10"},
+				{start: 8388608, stop: 11534336, fileId: "5,02982f80de50"},
+				{start: 11534336, stop: 14376529, fileId: "7,0299ad723803"},
+			},
+		},
+		// case 8: real bug
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 77824, FileId: "4,0b3df938e301", Mtime: 123},
+				{Offset: 471040, Size: 472225 - 471040, FileId: "6,0b3e0650019c", Mtime: 130},
+				{Offset: 77824, Size: 208896 - 77824, FileId: "4,0b3f0c7202f0", Mtime: 140},
+				{Offset: 208896, Size: 339968 - 208896, FileId: "2,0b4031a72689", Mtime: 150},
+				{Offset: 339968, Size: 471040 - 339968, FileId: "3,0b416a557362", Mtime: 160},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 77824, fileId: "4,0b3df938e301"},
+				{start: 77824, stop: 208896, fileId: "4,0b3f0c7202f0"},
+				{start: 208896, stop: 339968, fileId: "2,0b4031a72689"},
+				{start: 339968, stop: 471040, fileId: "3,0b416a557362"},
+				{start: 471040, stop: 472225, fileId: "6,0b3e0650019c"},
+			},
+		},
+	}
+
+	for i, testcase := range testcases {
+		log.Printf("++++++++++ merged test case %d ++++++++++++++++++++", i)
+		intervals := NonOverlappingVisibleIntervals(testcase.Chunks)
+		for x, interval := range intervals {
+			log.Printf("test case %d, interval %d, start=%d, stop=%d, fileId=%s",
+				i, x, interval.start, interval.stop, interval.fileId)
+		}
+		for x, interval := range intervals {
+			if interval.start != testcase.Expected[x].start {
+				t.Fatalf("failed on test case %d, interval %d, start %d, expect %d",
+					i, x, interval.start, testcase.Expected[x].start)
+			}
+			if interval.stop != testcase.Expected[x].stop {
+				t.Fatalf("failed on test case %d, interval %d, stop %d, expect %d",
+					i, x, interval.stop, testcase.Expected[x].stop)
+			}
+			if interval.fileId != testcase.Expected[x].fileId {
+				t.Fatalf("failed on test case %d, interval %d, chunkId %s, expect %s",
+					i, x, interval.fileId, testcase.Expected[x].fileId)
+			}
+		}
+		if len(intervals) != len(testcase.Expected) {
+			t.Fatalf("failed to compact test case %d, len %d expected %d", i, len(intervals), len(testcase.Expected))
+		}
+
+	}
+
+}
+
+func TestChunksReading(t *testing.T) {
+
+	testcases := []struct {
+		Chunks   []*filer_pb.FileChunk
+		Offset   int64
+		Size     int
+		Expected []*ChunkView
+	}{
+		// case 0: normal
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 100, Size: 100, FileId: "asdf", Mtime: 134},
+				{Offset: 200, Size: 100, FileId: "fsad", Mtime: 353},
+			},
+			Offset: 0,
+			Size:   250,
+			Expected: []*ChunkView{
+				{Offset: 0, Size: 100, FileId: "abc", LogicOffset: 0},
+				{Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 100},
+				{Offset: 0, Size: 50, FileId: "fsad", LogicOffset: 200},
+			},
+		},
+		// case 1: updates overwrite full chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
+			},
+			Offset: 50,
+			Size:   100,
+			Expected: []*ChunkView{
+				{Offset: 50, Size: 100, FileId: "asdf", LogicOffset: 50},
+			},
+		},
+		// case 2: updates overwrite part of previous chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 50, FileId: "asdf", Mtime: 134},
+			},
+			Offset: 25,
+			Size:   50,
+			Expected: []*ChunkView{
+				{Offset: 25, Size: 25, FileId: "asdf", LogicOffset: 25},
+				{Offset: 0, Size: 25, FileId: "abc", LogicOffset: 50},
+			},
+		},
+		// case 3: updates overwrite full chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
+				{Offset: 50, Size: 250, FileId: "xxxx", Mtime: 154},
+			},
+			Offset: 0,
+			Size:   200,
+			Expected: []*ChunkView{
+				{Offset: 0, Size: 50, FileId: "asdf", LogicOffset: 0},
+				{Offset: 0, Size: 150, FileId: "xxxx", LogicOffset: 50},
+			},
+		},
+		// case 4: updates far away from prev chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
+				{Offset: 250, Size: 250, FileId: "xxxx", Mtime: 154},
+			},
+			Offset: 0,
+			Size:   400,
+			Expected: []*ChunkView{
+				{Offset: 0, Size: 200, FileId: "asdf", LogicOffset: 0},
+				// {Offset: 0, Size: 150, FileId: "xxxx"}, // missing intervals should not happen
+			},
+		},
+		// case 5: updates overwrite full chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 184},
+				{Offset: 70, Size: 150, FileId: "abc", Mtime: 143},
+				{Offset: 80, Size: 100, FileId: "xxxx", Mtime: 134},
+			},
+			Offset: 0,
+			Size:   220,
+			Expected: []*ChunkView{
+				{Offset: 0, Size: 200, FileId: "asdf", LogicOffset: 0},
+				{Offset: 0, Size: 20, FileId: "abc", LogicOffset: 200},
+			},
+		},
+		// case 6: same updates
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+			},
+			Offset: 0,
+			Size:   100,
+			Expected: []*ChunkView{
+				{Offset: 0, Size: 100, FileId: "abc", LogicOffset: 0},
+			},
+		},
+		// case 7: edge cases
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 100, Size: 100, FileId: "asdf", Mtime: 134},
+				{Offset: 200, Size: 100, FileId: "fsad", Mtime: 353},
+			},
+			Offset: 0,
+			Size:   200,
+			Expected: []*ChunkView{
+				{Offset: 0, Size: 100, FileId: "abc", LogicOffset: 0},
+				{Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 100},
+			},
+		},
+	}
+
+	for i, testcase := range testcases {
+		log.Printf("++++++++++ read test case %d ++++++++++++++++++++", i)
+		chunks := ViewFromChunks(testcase.Chunks, testcase.Offset, testcase.Size)
+		for x, chunk := range chunks {
+			log.Printf("read case %d, chunk %d, offset=%d, size=%d, fileId=%s",
+				i, x, chunk.Offset, chunk.Size, chunk.FileId)
+			if chunk.Offset != testcase.Expected[x].Offset {
+				t.Fatalf("failed on read case %d, chunk %d, Offset %d, expect %d",
+					i, x, chunk.Offset, testcase.Expected[x].Offset)
+			}
+			if chunk.Size != testcase.Expected[x].Size {
+				t.Fatalf("failed on read case %d, chunk %d, Size %d, expect %d",
+					i, x, chunk.Size, testcase.Expected[x].Size)
+			}
+			if chunk.FileId != testcase.Expected[x].FileId {
+				t.Fatalf("failed on read case %d, chunk %d, FileId %s, expect %s",
+					i, x, chunk.FileId, testcase.Expected[x].FileId)
+			}
+			if chunk.LogicOffset != testcase.Expected[x].LogicOffset {
+				t.Fatalf("failed on read case %d, chunk %d, LogicOffset %d, expect %d",
+					i, x, chunk.LogicOffset, testcase.Expected[x].LogicOffset)
+			}
+		}
+		if len(chunks) != len(testcase.Expected) {
+			t.Fatalf("failed to read test case %d, len %d expected %d", i, len(chunks), len(testcase.Expected))
+		}
+	}
+
+}
+
+func BenchmarkCompactFileChunks(b *testing.B) {
+
+	var chunks []*filer_pb.FileChunk
+
+	k := 1024
+
+	for n := 0; n < k; n++ {
+		chunks = append(chunks, &filer_pb.FileChunk{
+			Offset: int64(n * 100), Size: 100, FileId: fmt.Sprintf("fileId%d", n), Mtime: int64(n),
+		})
+		chunks = append(chunks, &filer_pb.FileChunk{
+			Offset: int64(n * 50), Size: 100, FileId: fmt.Sprintf("fileId%d", n+k), Mtime: int64(n + k),
+		})
+	}
+
+	for n := 0; n < b.N; n++ {
+		CompactFileChunks(chunks)
+	}
+}

weed/filer2/filer.go

@@ -0,0 +1,306 @@
+package filer2
+
+import (
+	"context"
+	"fmt"
+	"google.golang.org/grpc"
+	"math"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
+	"github.com/karlseguin/ccache"
+)
+
+var (
+	OS_UID = uint32(os.Getuid())
+	OS_GID = uint32(os.Getgid())
+)
+
+type Filer struct {
+	store              FilerStore
+	directoryCache     *ccache.Cache
+	MasterClient       *wdclient.MasterClient
+	fileIdDeletionChan chan string
+	GrpcDialOption     grpc.DialOption
+}
+
+func NewFiler(masters []string, grpcDialOption grpc.DialOption) *Filer {
+	f := &Filer{
+		directoryCache:     ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
+		MasterClient:       wdclient.NewMasterClient(context.Background(), grpcDialOption, "filer", masters),
+		fileIdDeletionChan: make(chan string, 4096),
+		GrpcDialOption:     grpcDialOption,
+	}
+
+	go f.loopProcessingDeletion()
+
+	return f
+}
+
+func (f *Filer) SetStore(store FilerStore) {
+	f.store = store
+}
+
+func (f *Filer) DisableDirectoryCache() {
+	f.directoryCache = nil
+}
+
+func (fs *Filer) GetMaster() string {
+	return fs.MasterClient.GetMaster()
+}
+
+func (fs *Filer) KeepConnectedToMaster() {
+	fs.MasterClient.KeepConnectedToMaster()
+}
+
+func (f *Filer) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return f.store.BeginTransaction(ctx)
+}
+
+func (f *Filer) CommitTransaction(ctx context.Context) error {
+	return f.store.CommitTransaction(ctx)
+}
+
+func (f *Filer) RollbackTransaction(ctx context.Context) error {
+	return f.store.RollbackTransaction(ctx)
+}
+
+func (f *Filer) CreateEntry(ctx context.Context, entry *Entry) error {
+
+	if string(entry.FullPath) == "/" {
+		return nil
+	}
+
+	dirParts := strings.Split(string(entry.FullPath), "/")
+
+	// fmt.Printf("directory parts: %+v\n", dirParts)
+
+	var lastDirectoryEntry *Entry
+
+	for i := 1; i < len(dirParts); i++ {
+		dirPath := "/" + filepath.ToSlash(filepath.Join(dirParts[:i]...))
+		// fmt.Printf("%d directory: %+v\n", i, dirPath)
+
+		// first check local cache
+		dirEntry := f.cacheGetDirectory(dirPath)
+
+		// not found, check the store directly
+		if dirEntry == nil {
+			glog.V(4).Infof("find uncached directory: %s", dirPath)
+			dirEntry, _ = f.FindEntry(ctx, FullPath(dirPath))
+		} else {
+			glog.V(4).Infof("found cached directory: %s", dirPath)
+		}
+
+		// no such existing directory
+		if dirEntry == nil {
+
+			// create the directory
+			now := time.Now()
+
+			dirEntry = &Entry{
+				FullPath: FullPath(dirPath),
+				Attr: Attr{
+					Mtime:  now,
+					Crtime: now,
+					Mode:   os.ModeDir | 0770,
+					Uid:    entry.Uid,
+					Gid:    entry.Gid,
+				},
+			}
+
+			glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
+			mkdirErr := f.store.InsertEntry(ctx, dirEntry)
+			if mkdirErr != nil {
+				if _, err := f.FindEntry(ctx, FullPath(dirPath)); err == ErrNotFound {
+					return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
+				}
+			} else {
+				f.NotifyUpdateEvent(nil, dirEntry, false)
+			}
+
+		} else if !dirEntry.IsDirectory() {
+			return fmt.Errorf("%s is a file", dirPath)
+		}
+
+		// cache the directory entry
+		f.cacheSetDirectory(dirPath, dirEntry, i)
+
+		// remember the direct parent directory entry
+		if i == len(dirParts)-1 {
+			lastDirectoryEntry = dirEntry
+		}
+
+	}
+
+	if lastDirectoryEntry == nil {
+		return fmt.Errorf("parent folder not found: %v", entry.FullPath)
+	}
+
+	/*
+		if !hasWritePermission(lastDirectoryEntry, entry) {
+			glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d",
+				lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid)
+			return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
+		}
+	*/
+
+	oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
+
+	if oldEntry == nil {
+		if err := f.store.InsertEntry(ctx, entry); err != nil {
+			return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
+		}
+	} else {
+		if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
+			return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
+		}
+	}
+
+	f.NotifyUpdateEvent(oldEntry, entry, true)
+
+	f.deleteChunksIfNotNew(oldEntry, entry)
+
+	return nil
+}
+
+func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) {
+	if oldEntry != nil {
+		if oldEntry.IsDirectory() && !entry.IsDirectory() {
+			return fmt.Errorf("existing %s is a directory", entry.FullPath)
+		}
+		if !oldEntry.IsDirectory() && entry.IsDirectory() {
+			return fmt.Errorf("existing %s is a file", entry.FullPath)
+		}
+	}
+	return f.store.UpdateEntry(ctx, entry)
+}
+
+func (f *Filer) FindEntry(ctx context.Context, p FullPath) (entry *Entry, err error) {
+
+	now := time.Now()
+
+	if string(p) == "/" {
+		return &Entry{
+			FullPath: p,
+			Attr: Attr{
+				Mtime:  now,
+				Crtime: now,
+				Mode:   os.ModeDir | 0755,
+				Uid:    OS_UID,
+				Gid:    OS_GID,
+			},
+		}, nil
+	}
+	return f.store.FindEntry(ctx, p)
+}
+
+func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecursive bool, shouldDeleteChunks bool) (err error) {
+	entry, err := f.FindEntry(ctx, p)
+	if err != nil {
+		return err
+	}
+
+	if entry.IsDirectory() {
+		limit := int(1)
+		if isRecursive {
+			limit = math.MaxInt32
+		}
+		lastFileName := ""
+		includeLastFile := false
+		for limit > 0 {
+			entries, err := f.ListDirectoryEntries(ctx, p, lastFileName, includeLastFile, 1024)
+			if err != nil {
+				return fmt.Errorf("list folder %s: %v", p, err)
+			}
+
+			if len(entries) == 0 {
+				break
+			}
+
+			if isRecursive {
+				for _, sub := range entries {
+					lastFileName = sub.Name()
+					err = f.DeleteEntryMetaAndData(ctx, sub.FullPath, isRecursive, shouldDeleteChunks)
+					if err != nil {
+						return err
+					}
+					limit--
+					if limit <= 0 {
+						break
+					}
+				}
+			}
+
+			if len(entries) < 1024 {
+				break
+			}
+		}
+
+		f.cacheDelDirectory(string(p))
+
+	}
+
+	if shouldDeleteChunks {
+		f.DeleteChunks(p, entry.Chunks)
+	}
+
+	if p == "/" {
+		return nil
+	}
+	glog.V(3).Infof("deleting entry %v", p)
+
+	f.NotifyUpdateEvent(entry, nil, shouldDeleteChunks)
+
+	return f.store.DeleteEntry(ctx, p)
+}
+
+func (f *Filer) ListDirectoryEntries(ctx context.Context, p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
+	if strings.HasSuffix(string(p), "/") && len(p) > 1 {
+		p = p[0 : len(p)-1]
+	}
+	return f.store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit)
+}
+
+func (f *Filer) cacheDelDirectory(dirpath string) {
+
+	if dirpath == "/" {
+		return
+	}
+
+	if f.directoryCache == nil {
+		return
+	}
+	f.directoryCache.Delete(dirpath)
+	return
+}
+
+func (f *Filer) cacheGetDirectory(dirpath string) *Entry {
+
+	if f.directoryCache == nil {
+		return nil
+	}
+	item := f.directoryCache.Get(dirpath)
+	if item == nil {
+		return nil
+	}
+	return item.Value().(*Entry)
+}
+
+func (f *Filer) cacheSetDirectory(dirpath string, dirEntry *Entry, level int) {
+
+	if f.directoryCache == nil {
+		return
+	}
+
+	minutes := 60
+	if level < 10 {
+		minutes -= level * 6
+	}
+
+	f.directoryCache.Set(dirpath, dirEntry, time.Duration(minutes)*time.Minute)
+}