Release 0.74

fix https://github.com/chrislusf/seaweedfs/issues/441

.gitignore

@@ -1,3 +1,79 @@
-go/weed/.goxc*
+.goxc*
+vendor
 tags
 *.swp
+### OSX template
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio
+
+*.iml
+
+## Directory-based project format:
+.idea/
+# if you remove the above rule, at least ignore the following:
+
+# User-specific stuff:
+# .idea/workspace.xml
+# .idea/tasks.xml
+# .idea/dictionaries
+
+# Sensitive or high-churn files:
+# .idea/dataSources.ids
+# .idea/dataSources.xml
+# .idea/sqlDataSources.xml
+# .idea/dynamic.xml
+# .idea/uiDesigner.xml
+
+# Gradle:
+# .idea/gradle.xml
+# .idea/libraries
+
+# Mongo Explorer plugin:
+# .idea/mongoSettings.xml
+
+## File-based project format:
+*.ipr
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+
+test_data
+build

.travis.yml

@@ -1,14 +1,44 @@
+sudo: false
 language: go
-go: 
- - 1.3
- - 1.4.1
- - tip
+go:
+- 1.5
+- 1.6
+- tip
 
 before_install:
- - export PATH=/home/travis/gopath/bin:$PATH
+- export PATH=/home/travis/gopath/bin:$PATH
 
 install:
- - go get ./go/...
- 
+- go get ./weed/...
+
 script:
- - go test ./go/...
+- go test ./weed/...
+
+before_deploy:
+  - make release
+deploy:
+  provider: releases
+  skip_cleanup: true
+  api_key:
+    secure: ERL986+ncQ8lwAJUYDrQ8s2/FxF/cyNIwJIFCqspnWxQgGNNyokET9HapmlPSxjpFRF0q6L2WCg9OY3mSVRq4oI6hg1igOQ12KlLyN71XSJ3c8w0Ay5ho48TQ9l3f3Iu97mntBCe9l0R9pnT8wj1VI8YJxloXwUMG2yeTjA9aBI=
+  file:
+  - build/linux_arm.tar.gz
+  - build/linux_arm64.tar.gz
+  - build/linux_386.tar.gz
+  - build/linux_amd64.tar.gz
+  - build/darwin_amd64.tar.gz
+  - build/windows_386.zip
+  - build/windows_amd64.zip
+  - build/freebsd_arm.tar.gz
+  - build/freebsd_amd64.tar.gz
+  - build/freebsd_386.tar.gz
+  - build/netbsd_arm.tar.gz
+  - build/netbsd_amd64.tar.gz
+  - build/netbsd_386.tar.gz
+  - build/openbsd_arm.tar.gz
+  - build/openbsd_amd64.tar.gz
+  - build/openbsd_386.tar.gz
+  on:
+    tags: true
+    repo: chrislusf/seaweedfs
+    go: tip

Dockerfile

@@ -1,21 +0,0 @@
-FROM progrium/busybox
-
-WORKDIR /opt/weed
-
-RUN opkg-install curl
-RUN echo insecure >> ~/.curlrc
-
-RUN \
-  curl -Lks https://bintray.com$(curl -Lk http://bintray.com/chrislusf/seaweedfs/seaweedfs/_latestVersion | grep linux_amd64.tar.gz | sed -n "/href/ s/.*href=['\"]\([^'\"]*\)['\"].*/\1/gp") | gunzip | tar -xf - -C /opt/weed/ && \
-    mv weed_*/* /bin && \
-  chmod +x ./bin/weed
-
-EXPOSE 8080
-EXPOSE 9333
-
-VOLUME /data
-
-ENV WEED_HOME /opt/weed
-ENV PATH ${PATH}:${WEED_HOME}/bin
-
-ENTRYPOINT ["weed"]

Dockerfile.go_build

@@ -1,5 +1,5 @@
 FROM cydev/go
-RUN go get github.com/chrislusf/seaweedfs/go/weed
+RUN go get github.com/chrislusf/seaweedfs/weed
 EXPOSE 8080
 EXPOSE 9333
 VOLUME /data

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2016 Chris Lu
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -0,0 +1,113 @@
+BINARY = weed/weed
+package = github.com/chrislusf/seaweedfs/weed
+
+GO_FLAGS = #-v
+SOURCE_DIR = ./weed/
+
+appname := weed
+
+sources := $(wildcard *.go)
+
+build = GOOS=$(1) GOARCH=$(2) go build -o build/$(appname)$(3) $(SOURCE_DIR)
+tar = cd build && tar -cvzf $(1)_$(2).tar.gz $(appname)$(3) && rm $(appname)$(3)
+zip = cd build && zip $(1)_$(2).zip $(appname)$(3) && rm $(appname)$(3)
+
+
+all: build
+
+.PHONY : clean deps build linux release windows_build darwin_build linux_build bsd_build clean
+
+clean:
+	go clean -i $(GO_FLAGS) $(SOURCE_DIR)
+	rm -f $(BINARY)
+	rm -rf build/
+
+deps:
+	go get $(GO_FLAGS) -d $(SOURCE_DIR)
+
+build: deps
+	go build $(GO_FLAGS) -o $(BINARY) $(SOURCE_DIR)
+
+linux: deps
+	mkdir -p linux
+	GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -o linux/$(BINARY) $(SOURCE_DIR)
+
+release: deps windows_build darwin_build linux_build bsd_build
+
+##### LINUX BUILDS #####
+linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz
+
+build/linux_386.tar.gz: $(sources)
+	$(call build,linux,386,)
+	$(call tar,linux,386)
+
+build/linux_amd64.tar.gz: $(sources)
+	$(call build,linux,amd64,)
+	$(call tar,linux,amd64)
+
+build/linux_arm.tar.gz: $(sources)
+	$(call build,linux,arm,)
+	$(call tar,linux,arm)
+
+build/linux_arm64.tar.gz: $(sources)
+	$(call build,linux,arm64,)
+	$(call tar,linux,arm64)
+
+##### DARWIN (MAC) BUILDS #####
+darwin_build: build/darwin_amd64.tar.gz
+
+build/darwin_amd64.tar.gz: $(sources)
+	$(call build,darwin,amd64,)
+	$(call tar,darwin,amd64)
+
+##### WINDOWS BUILDS #####
+windows_build: build/windows_386.zip build/windows_amd64.zip
+
+build/windows_386.zip: $(sources)
+	$(call build,windows,386,.exe)
+	$(call zip,windows,386,.exe)
+
+build/windows_amd64.zip: $(sources)
+	$(call build,windows,amd64,.exe)
+	$(call zip,windows,amd64,.exe)
+
+##### BSD BUILDS #####
+bsd_build: build/freebsd_arm.tar.gz build/freebsd_386.tar.gz build/freebsd_amd64.tar.gz \
+ build/netbsd_arm.tar.gz build/netbsd_386.tar.gz build/netbsd_amd64.tar.gz \
+ build/openbsd_arm.tar.gz build/openbsd_386.tar.gz build/openbsd_amd64.tar.gz
+
+build/freebsd_386.tar.gz: $(sources)
+	$(call build,freebsd,386,)
+	$(call tar,freebsd,386)
+
+build/freebsd_amd64.tar.gz: $(sources)
+	$(call build,freebsd,amd64,)
+	$(call tar,freebsd,amd64)
+
+build/freebsd_arm.tar.gz: $(sources)
+	$(call build,freebsd,arm,)
+	$(call tar,freebsd,arm)
+
+build/netbsd_386.tar.gz: $(sources)
+	$(call build,netbsd,386,)
+	$(call tar,netbsd,386)
+
+build/netbsd_amd64.tar.gz: $(sources)
+	$(call build,netbsd,amd64,)
+	$(call tar,netbsd,amd64)
+
+build/netbsd_arm.tar.gz: $(sources)
+	$(call build,netbsd,arm,)
+	$(call tar,netbsd,arm)
+
+build/openbsd_386.tar.gz: $(sources)
+	$(call build,openbsd,386,)
+	$(call tar,openbsd,386)
+
+build/openbsd_amd64.tar.gz: $(sources)
+	$(call build,openbsd,amd64,)
+	$(call tar,openbsd,amd64)
+
+build/openbsd_arm.tar.gz: $(sources)
+	$(call build,openbsd,arm,)
+	$(call tar,openbsd,arm)

README.md

@@ -1,11 +1,19 @@
 # SeaweedFS
 
 [![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs)
-[![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/go?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/go)
+[![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
 
 ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)
 
+
+- [Download Binaries for different platforms](https://github.com/chrislusf/seaweedfs/releases/latest)
+- [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
+- [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
+
+
+[![](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=EEECLJ8QGTTPC) 
+
 ## Introduction
 
 SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
@@ -13,28 +21,23 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two
 1. to store billions of files!
 2. to serve the files fast!
 
-Instead of supporting full POSIX file system semantics, SeaweedFS choose to implement only a key~file mapping. Similar to the word "NoSQL", you can call it as "NoFS".
+Instead of supporting full POSIX file system semantics, SeaweedFS choose to implement only a key->file mapping. Similar to "NoSQL", you might call it "NoFS".
 
-Instead of managing all file metadata in a central master, SeaweedFS choose to manages file volumes in the central master, and let volume servers manage files and the metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers' memories, allowing faster file access with just one disk read operation!
+Instead of managing all file metadata in a central master, the central master only manages file volumes, and it lets these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (just one disk read operation).
 
-SeaweedFS costs only 40 bytes disk storage for each file's metadata. It is so simple with O(1) disk read that you are welcome to challenge the performance with your actual use cases.
+There is only a 40 bytes disk storage overhead for each file's metadata. It is so simple with O(1) disk read that you are welcome to challenge the performance with your actual use cases.
 
-SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). It grows with more features along the way.
-
-![](https://api.bintray.com/packages/chrislusf/seaweedfs/seaweedfs/images/download.png)
-https://bintray.com/chrislusf/seaweedfs/seaweedfs Download latest compiled binaries for different platforms here.
-
-https://groups.google.com/d/forum/seaweedfs SeaweedFS Discussion Group
+SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). SeaweedFS is currently growing, with more features on the way.
 
 ## Additional Features
 * Can choose no replication or different replication level, rack and data center aware
-* Automatic master servers failover. No single point of failure, SPOF.
+* Automatic master servers failover - no single point of failure (SPOF)
 * Automatic Gzip compression depending on file mime type
 * Automatic compaction to reclaimed disk spaces after deletion or update
-* Servers in the same cluster can have different disk spaces, different file systems, different OS
-* Adding/Removing servers do not cause any data re-balancing
+* Servers in the same cluster can have different disk spaces, file systems, OS etc.
+* Adding/Removing servers does **not** cause any data re-balancing
 * Optional [filer server][Filer] provides "normal" directories and files via http
-* For jpeg pictures, optionally fix the orientation.
+* Optionally fix the orientation for jpeg pictures
 * Support Etag, Accept-Range, Last-Modified, etc.
 * Support in-memory/leveldb/boltdb mode tuning for memory/performance balance.
 
@@ -42,15 +45,14 @@ https://groups.google.com/d/forum/seaweedfs SeaweedFS Discussion Group
 
 ## Example Usage
 By default, the master node runs on port 9333, and the volume nodes runs on port 8080.
-Here I will start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. Here I just use localhost as example.
+Here I will start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. I just use localhost as an example.
 
-SeaweedFS uses HTTP REST operations to write, read, delete. The return results are JSON or JSONP format.
+SeaweedFS uses HTTP REST operations to write, read, delete. The responses are in JSON or JSONP format.
 
 ### Start Master Server
 
 ```
 > ./weed master
-
 ```
 
 ### Start Volume Servers ###
@@ -58,26 +60,26 @@ SeaweedFS uses HTTP REST operations to write, read, delete. The return results a
 ```
 > weed volume -dir="/tmp/data1" -max=5  -mserver="localhost:9333" -port=8080 &
 > weed volume -dir="/tmp/data2" -max=10 -mserver="localhost:9333" -port=8081 &
-
 ```
 
+
 ### Write File ###
 
-To upload a file, first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an fid and a volume server url:
+To upload a file: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server url:
 
 ```
 > curl -X POST http://localhost:9333/dir/assign
 {"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
 ```
 
-Second, to store the file content, send a HTTP multipart PUT or POST request to `url + '/' + fid` from the response:
+Second, to store the file content, send a HTTP multi-part PUT or POST request to `url + '/' + fid` from the response:
 
 ```
 > curl -X PUT -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6
 {"size": 43234}
 ```
 
-For update, send another PUT or POST request with updated file content.
+To update, send another PUT or POST request with updated file content.
 
 For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
 
@@ -85,21 +87,22 @@ For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
 > curl -X DELETE http://127.0.0.1:8080/3,01637037d6
 ```
 ### Save File Id ###
-Now you can save the fid, 3,01637037d6 in this case, to some database field.
 
-The number 3 here, is a volume id. After the comma, it's one file key, 01, and a file cookie, 637037d6.
+Now you can save the `fid`, 3,01637037d6 in this case, to a database field.
 
-The volume id is an unsigned 32 bit integer. The file key is an unsigned 64bit integer. The file cookie is an unsigned 32bit integer, used to prevent URL guessing.
+The number 3 at the start represents a volume id. After the comma, it's one file key, 01, and a file cookie, 637037d6.
 
-The file key and file cookie are both coded in hex. You can store the <volume id, file key, file cookie> tuple in your own format, or simply store the fid as string.
+The volume id is an unsigned 32-bit integer. The file key is an unsigned 64-bit integer. The file cookie is an unsigned 32-bit integer, used to prevent URL guessing.
+
+The file key and file cookie are both coded in hex. You can store the <volume id, file key, file cookie> tuple in your own format, or simply store the `fid` as string.
 
 If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most usage would not need 2^32 volumes.
 
-If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, 4-byte integer for file cookie. So 16 bytes are enough (more than enough).
+If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, 4-byte integer for file cookie. So 16 bytes are more than enough.
 
 ### Read File ###
 
-Here is the example on how to render the URL.
+Here is an example of how to render the URL.
 
 First lookup the volume server's URLs by the file's volumeId:
 
@@ -108,7 +111,7 @@ First lookup the volume server's URLs by the file's volumeId:
 {"locations":[{"publicUrl":"localhost:8080","url":"localhost:8080"}]}
 ```
 
-(However, since usually there are not too many volume servers, and volumes does not move often, you can cache the results most of the time. Depends on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read.)
+Since (usually) there are not too many volume servers, and volumes don't move often, you can cache the results most of the time. Depends on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read.)
 
 Now you can take the public url, render the url or directly read from the volume server via url:
 
@@ -116,7 +119,7 @@ Now you can take the public url, render the url or directly read from the volume
  http://localhost:8080/3,01637037d6.jpg
 ```
 
-Notice we add an file extension ".jpg" here. It's optional and just one way for the client to specify the file content type.
+Notice we add a file extension ".jpg" here. It's optional and just one way for the client to specify the file content type.
 
 If you want a nicer URL, you can use one of these alternative URL formats:
 
@@ -128,14 +131,21 @@ If you want a nicer URL, you can use one of these alternative URL formats:
  http://localhost:8080/3,01637037d6
 ```
 
+If you want get a scaled version of an image, you can add some params:
+
+```
+http://localhost:8080/3/01637037d6.jpg?height=200&width=200
+```
+
 ### Rack-Aware and Data Center-Aware Replication ###
-SeaweedFS apply the replication strategy on a volume level. So when you are getting a file id, you can specify the replication strategy. For example:
+
+SeaweedFS applies the replication strategy at a volume level. So, when you are getting a file id, you can specify the replication strategy. For example:
 
 ```
 curl -X POST http://localhost:9333/dir/assign?replication=001
 ```
 
-Here is the meaning of the replication parameter:
+The replication parameter options are:
 
 ```
 000: no replication
@@ -161,9 +171,7 @@ Volume servers can start with a specific data center name:
  weed volume -dir=/tmp/2 -port=8081 -dataCenter=dc2
 ```
 
-Or the master server can determine the data center via volume server's IP address and settings in weed.conf file.
-
-Now when requesting a file key, an optional "dataCenter" parameter can limit the assigned volume to the specific data center. For example, this specifies that the assigned volume should be limited to 'dc1':
+When requesting a file key, an optional "dataCenter" parameter can limit the assigned volume to the specific data center. For example, this specifies that the assigned volume should be limited to 'dc1':
 
 ```
  http://localhost:9333/dir/assign?dataCenter=dc1
@@ -181,63 +189,70 @@ Now when requesting a file key, an optional "dataCenter" parameter can limit the
 [feat-4]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#collection-as-a-simple-name-space
 
 ## Architecture ##
-Usually distributed file system split each file into chunks, and a central master keeps a mapping of a filename and a chunk index to chunk handles, and also which chunks each chunk server has.
 
-This has the draw back that the central master can not handle many small files efficiently, and since all read requests need to go through the chunk master, responses would be slow for many concurrent web users.
+Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.
 
-Instead of managing chunks, SeaweedFS choose to manage data volumes in the master server. Each data volume is size 32GB, and can hold a lot of files. And each storage node can has many data volumes. So the master node only needs to store the metadata about the volumes, which is fairly small amount of data and pretty static most of the time.
+The main drawback is that the central master can't handle many small files efficiently, and since all read requests need to go through the chunk master, might not scale well for many concurrent users.
 
-The actual file metadata is stored in each volume on volume servers. Since each volume server only manage metadata of files on its own disk, and only 16 bytes for each file, all file access can read file metadata just from memory and only needs one disk operation to actually read file data.
+Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is size 32GB, and can hold a lot of files. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is fairly small amount of data and is generally stable.
+
+The actual file metadata is stored in each volume on volume servers. Since each volume server only manages metadata of files on its own disk, with only 16 bytes for each file, all file access can read file metadata just from memory and only needs one disk operation to actually read file data.
 
 For comparison, consider that an xfs inode structure in Linux is 536 bytes.
 
 ### Master Server and Volume Server ###
+
 The architecture is fairly simple. The actual data is stored in volumes on storage nodes. One volume server can have multiple volumes, and can both support read and write access with basic authentication.
 
 All volumes are managed by a master server. The master server contains volume id to volume server mapping. This is fairly static information, and could be cached easily.
 
-On each write request, the master server also generates a file key, which is a growing 64bit unsigned integer. Since the write requests are not as busy as read requests, one master server should be able to handle the concurrency well.
-
+On each write request, the master server also generates a file key, which is a growing 64-bit unsigned integer. Since write requests are not generally as frequent as read requests, one master server should be able to handle the concurrency well.
 
 ### Write and Read files ###
 
-When a client sends a write request, the master server returns <volume id, file key, file cookie, volume node url> for the file. The client then contact the volume node and POST the file content via REST.
+When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node url) for the file. The client then contacts the volume node and POSTs the file content.
 
-When a client needs to read a file based on <volume id, file key, file cookie>, it can ask the master server by the <volum id> for the <volume node url, volume node public url>, or from cache. Then the client can HTTP GET the content via REST, or just render the URL on web pages and let browsers to fetch the content.
+When a client needs to read a file based on (volume id, file key, file cookie), it can ask the master server by the volume id for the (volume node url, volume node public url), or retrieve this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
 
-Please see the example for details on write-read process.
+Please see the example for details on the write-read process.
 
 ### Storage Size ###
-In current implementation, each volume can be size of 8x2^32 bytes (32GiB). This is because of aligning contents to 8 bytes. We can be easily increased to 64G, or 128G, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
+
+In the current implementation, each volume can be 8x2^32 bytes (32GiB). This is because of we align content to 8 bytes. We can easily increase this to 64G, or 128G, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
 
 There can be 2^32 volumes. So total system size is 8 x 2^32 bytes x 2^32 = 8 x 4GiB x 4Gi = 128EiB (2^67 bytes, or 128 exbibytes).
 
 Each individual file size is limited to the volume size.
 
 ### Saving memory ###
+
 All file meta information on volume server is readable from memory without disk access. Each file just takes an 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own the space cost for the map. But usually the disk runs out before the memory does.
 
 
-## Compared to Other File Systems##
-Frankly, I don't use other distributed file systems too often. All seems more complicated than necessary. Please correct me if anything here is wrong.
+## Compared to Other File Systems ##
+
+Most other distributed file systems seem more complicated than necessary.
 
 ### Compared to Ceph ###
-Ceph can be setup similar to SeaweedFS as a key~blob store. It is much more complicated, with the need to support layers on top of it. Here is a more detailed comparison. https://github.com/chrislusf/seaweedfs/issues/120
 
-SeaweedFS is meant to be fast and simple, both during usage and during setup. If you do not understand how it works when you reach here, we failed! Jokes aside, you should not need any consulting service for it.
+Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/chrislusf/seaweedfs/issues/120)
+
+SeaweedFS is meant to be fast and simple, both during usage and during setup. If you do not understand how it works when you reach here, we failed! Please raise an issue with any questions or update this file with clarifications.
 
 SeaweedFS has a centralized master to lookup free volumes, while Ceph uses hashing to locate its objects. Having a centralized master makes it easy to code and manage. HDFS/GFS has the single name node for years. SeaweedFS now support multiple master nodes.
 
 Ceph hashing avoids SPOF, but makes it complicated when moving or adding servers.
 
 ### Compared to HDFS ###
+
 HDFS uses the chunk approach for each file, and is ideal for streaming large files.
 
 SeaweedFS is ideal for serving relatively smaller files quickly and concurrently.
 
 SeaweedFS can also store extra large files by splitting them into manageable data chunks, and store the file ids of the data chunks into a meta chunk. This is managed by "weed upload/download" tool, and the weed master or volume servers are agnostic about it.
 
-### Compared to MogileFS###
+### Compared to MogileFS ###
+
 SeaweedFS has 2 components: directory server, storage nodes.
 
 MogileFS has 3 components: tracers, database, storage nodes.
@@ -245,16 +260,18 @@ MogileFS has 3 components: tracers, database, storage nodes.
 One more layer means slower access, more operation complexity, more failure possibility.
 
 ### Compared to GlusterFS ###
-SeaweedFS is not POSIX compliant, and has simple implementation.
 
-GlusterFS is POSIX compliant, much more complex.
+SeaweedFS is not POSIX compliant, and has a simple implementation.
 
-### Compared to Mongo's GridFS ###
-Mongo's GridFS splits files into chunks and manage chunks in the central mongodb. For every read or write request, the database needs to query the metadata. It's OK if this is not a bottleneck yet, but for a lot of concurrent reads this unnecessary query could slow things down.
+GlusterFS is POSIX compliant, and is much more complex.
 
-Since files are chunked(default to 256KB), there will be multiple metadata readings and multiple chunk readings, linear to the file size. One  2.56MB file would require at least 20 disk read requests.
+### Compared to MongoDB's GridFS ###
 
-On the contrary, SeaweedFS uses large file volume of 32G size to store lots of files, and only manages file volumes in the master server. Each volume manages file metadata themselves. So all the file metadata is spread onto the volume nodes memories, and just one disk read is needed.
+Mongo's GridFS splits files into chunks and manage chunks in the central MongoDB. For every read or write request, the database needs to query the metadata. It's OK if this is not yet a bottleneck, but for a lot of concurrent reads this unnecessary query could slow things down.
+
+Since files are chunked(default to 256KB), there will be multiple metadata readings and multiple chunk readings, linear to the file size. One 2.56MB file would require at least 20 disk read requests.
+
+On the contrary, SeaweedFS uses large file volume of 32G size to store lots of files, and only manages file volumes in the master server. Each volume manages file metadata itself. So all file metadata is spread across the volume nodes, and just one disk read is needed.
 
 ## Dev plan ##
 
@@ -279,15 +296,16 @@ http://mercurial.selenic.com/downloads
 
 step 3: download, compile, and install the project by executing the following command
 
-go get github.com/chrislusf/seaweedfs/go/weed
+go get github.com/chrislusf/seaweedfs/weed
 
 once this is done, you should see the executable "weed" under $GOPATH/bin
 
-step 4: after you modify your code locally, you could start a local build by calling "go install" under $GOPATH/src/github.com/chrislusf/seaweedfs/go/weed
+step 4: after you modify your code locally, you could start a local build by calling "go install" under $GOPATH/src/github.com/chrislusf/seaweedfs/weed
 
 ## Disk Related topics ##
 
 ### Hard Drive Performance ###
+
 When testing read performance on SeaweedFS, it basically becomes performance test your hard drive's random read speed. Hard Drive usually get 100MB/s~200MB/s.
 
 ### Solid State Disk
@@ -298,61 +316,57 @@ To modify or delete small files, SSD must delete a whole block at a time, and mo
 
 POSIX support
 
-
 ## Benchmark
 
-My Own Unscientific Single Machine Results on Mac Book with Solid State Disk, CPU: 1 Intel Core i7 2.2GHz.
+My Own Unscientific Single Machine Results on Mac Book with Solid State Disk, CPU: 1 Intel Core i7 2.6GHz.
 
 Write 1 million 1KB file:
 ```
-Concurrency Level:      64
-Time taken for tests:   182.456 seconds
+Concurrency Level:      16
+Time taken for tests:   88.796 seconds
 Complete requests:      1048576
 Failed requests:        0
-Total transferred:      1073741824 bytes
-Requests per second:    5747.01 [#/sec]
-Transfer rate:          5747.01 [Kbytes/sec]
+Total transferred:      1106764659 bytes
+Requests per second:    11808.87 [#/sec]
+Transfer rate:          12172.05 [Kbytes/sec]
 
 Connection Times (ms)
               min      avg        max      std
-Total:        0.3      10.9       430.9      5.7
+Total:        0.2      1.3       44.8      0.9
 
 Percentage of the requests served within a certain time (ms)
-   50%     10.2 ms
-   66%     12.0 ms
-   75%     12.6 ms
-   80%     12.9 ms
-   90%     14.0 ms
-   95%     14.9 ms
-   98%     16.2 ms
-   99%     17.3 ms
-  100%    430.9 ms
+   50%      1.1 ms
+   66%      1.3 ms
+   75%      1.5 ms
+   80%      1.7 ms
+   90%      2.1 ms
+   95%      2.6 ms
+   98%      3.7 ms
+   99%      4.6 ms
+  100%     44.8 ms
 ```
 
 Randomly read 1 million files:
 ```
-Concurrency Level:      64
-Time taken for tests:   80.732 seconds
+Concurrency Level:      16
+Time taken for tests:   34.263 seconds
 Complete requests:      1048576
 Failed requests:        0
-Total transferred:      1073741824 bytes
-Requests per second:    12988.37 [#/sec]
-Transfer rate:          12988.37 [Kbytes/sec]
+Total transferred:      1106762945 bytes
+Requests per second:    30603.34 [#/sec]
+Transfer rate:          31544.49 [Kbytes/sec]
 
 Connection Times (ms)
               min      avg        max      std
-Total:        0.0      4.7       254.3      6.3
+Total:        0.0      0.5       20.7      0.7
 
 Percentage of the requests served within a certain time (ms)
-   50%      2.6 ms
-   66%      2.9 ms
-   75%      3.7 ms
-   80%      4.7 ms
-   90%     10.3 ms
-   95%     16.6 ms
-   98%     26.3 ms
-   99%     34.8 ms
-  100%    254.3 ms
+   50%      0.4 ms
+   75%      0.5 ms
+   95%      0.6 ms
+   98%      0.8 ms
+   99%      1.2 ms
+  100%     20.7 ms
 ```
 
 

docker/Dockerfile

@@ -0,0 +1,17 @@
+FROM frolvlad/alpine-glibc:alpine-3.4
+
+RUN apk add --no-cache --virtual=build-dependencies --update curl wget ca-certificates && \
+    wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o '/chrislusf/seaweedfs/releases/download/.*/linux_amd64.tar.gz') && \
+    tar -C /usr/bin/ -xzvf /tmp/linux_amd64.tar.gz && \
+    apk del curl wget ca-certificates build-dependencies && \
+    rm -rf /tmp/*
+
+EXPOSE 8080
+EXPOSE 9333
+
+VOLUME /data
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/docker-compose.yml

@@ -0,0 +1,23 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs
+    ports:
+      - 9333:9333
+    command: "master"
+    networks:
+      default:
+        aliases:
+          - seaweed_master
+  volume:
+    image: chrislusf/seaweedfs
+    ports:
+      - 8080:8080
+    command: 'volume -max=5 -mserver="master:9333" -port=8080'
+    depends_on: 
+     - master
+    networks:
+      default:
+        aliases:
+          - seaweed_volume

docker/entrypoint.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+
+case "$1" in
+
+  'master')
+  	ARGS="-ip `hostname -i` -mdir /data"
+	# Is this instance linked with an other master? (Docker commandline "--link master1:master")
+	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
+		ARGS="$ARGS -peers=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
+	fi
+  	exec /usr/bin/weed $@ $ARGS
+	;;
+
+  'volume')
+  	ARGS="-ip `hostname -i` -dir /data"
+	# Is this instance linked with a master? (Docker commandline "--link master1:master")
+  	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
+		ARGS="$ARGS -mserver=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
+	fi
+  	exec /usr/bin/weed $@ $ARGS
+	;;
+
+  'server')
+  	ARGS="-ip `hostname -i` -dir /data"
+  	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
+		ARGS="$ARGS -master.peers=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
+	fi
+  	exec /usr/bin/weed $@ $ARGS
+  	;;
+
+  *)
+  	exec /usr/bin/weed $@
+	;;
+esac

go/operation/system_message.pb.go

@@ -1,203 +0,0 @@
-// Code generated by protoc-gen-go.
-// source: system_message.proto
-// DO NOT EDIT!
-
-/*
-Package operation is a generated protocol buffer package.
-
-It is generated from these files:
-	system_message.proto
-
-It has these top-level messages:
-	VolumeInformationMessage
-	JoinMessage
-*/
-package operation
-
-import proto "github.com/golang/protobuf/proto"
-import math "math"
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = math.Inf
-
-type VolumeInformationMessage struct {
-	Id               *uint32 `protobuf:"varint,1,req,name=id" json:"id,omitempty"`
-	Size             *uint64 `protobuf:"varint,2,req,name=size" json:"size,omitempty"`
-	Collection       *string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"`
-	FileCount        *uint64 `protobuf:"varint,4,req,name=file_count" json:"file_count,omitempty"`
-	DeleteCount      *uint64 `protobuf:"varint,5,req,name=delete_count" json:"delete_count,omitempty"`
-	DeletedByteCount *uint64 `protobuf:"varint,6,req,name=deleted_byte_count" json:"deleted_byte_count,omitempty"`
-	ReadOnly         *bool   `protobuf:"varint,7,opt,name=read_only" json:"read_only,omitempty"`
-	ReplicaPlacement *uint32 `protobuf:"varint,8,req,name=replica_placement" json:"replica_placement,omitempty"`
-	Version          *uint32 `protobuf:"varint,9,opt,name=version,def=2" json:"version,omitempty"`
-	Ttl              *uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"`
-	XXX_unrecognized []byte  `json:"-"`
-}
-
-func (m *VolumeInformationMessage) Reset()         { *m = VolumeInformationMessage{} }
-func (m *VolumeInformationMessage) String() string { return proto.CompactTextString(m) }
-func (*VolumeInformationMessage) ProtoMessage()    {}
-
-const Default_VolumeInformationMessage_Version uint32 = 2
-
-func (m *VolumeInformationMessage) GetId() uint32 {
-	if m != nil && m.Id != nil {
-		return *m.Id
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetSize() uint64 {
-	if m != nil && m.Size != nil {
-		return *m.Size
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetCollection() string {
-	if m != nil && m.Collection != nil {
-		return *m.Collection
-	}
-	return ""
-}
-
-func (m *VolumeInformationMessage) GetFileCount() uint64 {
-	if m != nil && m.FileCount != nil {
-		return *m.FileCount
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetDeleteCount() uint64 {
-	if m != nil && m.DeleteCount != nil {
-		return *m.DeleteCount
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetDeletedByteCount() uint64 {
-	if m != nil && m.DeletedByteCount != nil {
-		return *m.DeletedByteCount
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetReadOnly() bool {
-	if m != nil && m.ReadOnly != nil {
-		return *m.ReadOnly
-	}
-	return false
-}
-
-func (m *VolumeInformationMessage) GetReplicaPlacement() uint32 {
-	if m != nil && m.ReplicaPlacement != nil {
-		return *m.ReplicaPlacement
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetVersion() uint32 {
-	if m != nil && m.Version != nil {
-		return *m.Version
-	}
-	return Default_VolumeInformationMessage_Version
-}
-
-func (m *VolumeInformationMessage) GetTtl() uint32 {
-	if m != nil && m.Ttl != nil {
-		return *m.Ttl
-	}
-	return 0
-}
-
-type JoinMessage struct {
-	IsInit           *bool                       `protobuf:"varint,1,opt,name=is_init" json:"is_init,omitempty"`
-	Ip               *string                     `protobuf:"bytes,2,req,name=ip" json:"ip,omitempty"`
-	Port             *uint32                     `protobuf:"varint,3,req,name=port" json:"port,omitempty"`
-	PublicUrl        *string                     `protobuf:"bytes,4,opt,name=public_url" json:"public_url,omitempty"`
-	MaxVolumeCount   *uint32                     `protobuf:"varint,5,req,name=max_volume_count" json:"max_volume_count,omitempty"`
-	MaxFileKey       *uint64                     `protobuf:"varint,6,req,name=max_file_key" json:"max_file_key,omitempty"`
-	DataCenter       *string                     `protobuf:"bytes,7,opt,name=data_center" json:"data_center,omitempty"`
-	Rack             *string                     `protobuf:"bytes,8,opt,name=rack" json:"rack,omitempty"`
-	Volumes          []*VolumeInformationMessage `protobuf:"bytes,9,rep,name=volumes" json:"volumes,omitempty"`
-	AdminPort        *uint32                     `protobuf:"varint,10,opt,name=admin_port" json:"admin_port,omitempty"`
-	XXX_unrecognized []byte                      `json:"-"`
-}
-
-func (m *JoinMessage) Reset()         { *m = JoinMessage{} }
-func (m *JoinMessage) String() string { return proto.CompactTextString(m) }
-func (*JoinMessage) ProtoMessage()    {}
-
-func (m *JoinMessage) GetIsInit() bool {
-	if m != nil && m.IsInit != nil {
-		return *m.IsInit
-	}
-	return false
-}
-
-func (m *JoinMessage) GetIp() string {
-	if m != nil && m.Ip != nil {
-		return *m.Ip
-	}
-	return ""
-}
-
-func (m *JoinMessage) GetPort() uint32 {
-	if m != nil && m.Port != nil {
-		return *m.Port
-	}
-	return 0
-}
-
-func (m *JoinMessage) GetPublicUrl() string {
-	if m != nil && m.PublicUrl != nil {
-		return *m.PublicUrl
-	}
-	return ""
-}
-
-func (m *JoinMessage) GetMaxVolumeCount() uint32 {
-	if m != nil && m.MaxVolumeCount != nil {
-		return *m.MaxVolumeCount
-	}
-	return 0
-}
-
-func (m *JoinMessage) GetMaxFileKey() uint64 {
-	if m != nil && m.MaxFileKey != nil {
-		return *m.MaxFileKey
-	}
-	return 0
-}
-
-func (m *JoinMessage) GetDataCenter() string {
-	if m != nil && m.DataCenter != nil {
-		return *m.DataCenter
-	}
-	return ""
-}
-
-func (m *JoinMessage) GetRack() string {
-	if m != nil && m.Rack != nil {
-		return *m.Rack
-	}
-	return ""
-}
-
-func (m *JoinMessage) GetVolumes() []*VolumeInformationMessage {
-	if m != nil {
-		return m.Volumes
-	}
-	return nil
-}
-
-func (m *JoinMessage) GetAdminPort() uint32 {
-	if m != nil && m.AdminPort != nil {
-		return *m.AdminPort
-	}
-	return 0
-}
-
-func init() {
-}

go/operation/system_message_test.go

@@ -1,59 +0,0 @@
-package operation
-
-import (
-	"encoding/json"
-	"log"
-	"testing"
-
-	"github.com/golang/protobuf/proto"
-)
-
-func TestSerialDeserial(t *testing.T) {
-	volumeMessage := &VolumeInformationMessage{
-		Id:               proto.Uint32(12),
-		Size:             proto.Uint64(2341234),
-		Collection:       proto.String("benchmark"),
-		FileCount:        proto.Uint64(2341234),
-		DeleteCount:      proto.Uint64(234),
-		DeletedByteCount: proto.Uint64(21234),
-		ReadOnly:         proto.Bool(false),
-		ReplicaPlacement: proto.Uint32(210),
-		Version:          proto.Uint32(2),
-	}
-	var volumeMessages []*VolumeInformationMessage
-	volumeMessages = append(volumeMessages, volumeMessage)
-
-	joinMessage := &JoinMessage{
-		IsInit:         proto.Bool(true),
-		Ip:             proto.String("127.0.3.12"),
-		Port:           proto.Uint32(34546),
-		PublicUrl:      proto.String("localhost:2342"),
-		MaxVolumeCount: proto.Uint32(210),
-		MaxFileKey:     proto.Uint64(324234423),
-		DataCenter:     proto.String("dc1"),
-		Rack:           proto.String("rack2"),
-		Volumes:        volumeMessages,
-	}
-
-	data, err := proto.Marshal(joinMessage)
-	if err != nil {
-		log.Fatal("marshaling error: ", err)
-	}
-	newMessage := &JoinMessage{}
-	err = proto.Unmarshal(data, newMessage)
-	if err != nil {
-		log.Fatal("unmarshaling error: ", err)
-	}
-	log.Println("The pb data size is", len(data))
-
-	jsonData, jsonError := json.Marshal(joinMessage)
-	if jsonError != nil {
-		log.Fatal("json marshaling error: ", jsonError)
-	}
-	log.Println("The json data size is", len(jsonData), string(jsonData))
-
-	// Now test and newTest contain the same data.
-	if *joinMessage.PublicUrl != *newMessage.PublicUrl {
-		log.Fatalf("data mismatch %q != %q", *joinMessage.PublicUrl, *newMessage.PublicUrl)
-	}
-}

go/proto/Makefile

@@ -1,4 +0,0 @@
-TARG=../operation
-
-all:
-	protoc --go_out=$(TARG) system_message.proto

go/proto/system_message.proto

@@ -1,27 +0,0 @@
-package operation;
-
-message VolumeInformationMessage {
-  required uint32 id = 1;
-  required uint64 size = 2;
-  optional string collection = 3;
-  required uint64 file_count = 4;
-  required uint64 delete_count = 5;
-  required uint64 deleted_byte_count = 6;
-  optional bool read_only = 7;
-  required uint32 replica_placement = 8;
-  optional uint32 version = 9 [default=2];
-  optional uint32 ttl = 10;
-}
-
-message JoinMessage {
-  optional bool   is_init = 1;
-  required string ip = 2;
-  required uint32 port = 3;
-  optional string public_url = 4;
-  required uint32 max_volume_count = 5;
-  required uint64 max_file_key = 6;
-  optional string data_center = 7;
-  optional string rack = 8;
-  repeated VolumeInformationMessage volumes = 9;
-  optional uint32 admin_port = 10;
-}

go/storage/needle_map.go

@@ -1,68 +0,0 @@
-package storage
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/chrislusf/seaweedfs/go/util"
-)
-
-type NeedleMapType int
-
-const (
-	NeedleMapInMemory NeedleMapType = iota
-	NeedleMapLevelDb
-	NeedleMapBoltDb
-)
-
-type NeedleMapper interface {
-	Put(key uint64, offset uint32, size uint32) error
-	Get(key uint64) (element *NeedleValue, ok bool)
-	Delete(key uint64) error
-	Close()
-	Destroy() error
-	ContentSize() uint64
-	DeletedSize() uint64
-	FileCount() int
-	DeletedCount() int
-	MaxFileKey() uint64
-}
-
-type mapMetric struct {
-	DeletionCounter     int    `json:"DeletionCounter"`
-	FileCounter         int    `json:"FileCounter"`
-	DeletionByteCounter uint64 `json:"DeletionByteCounter"`
-	FileByteCounter     uint64 `json:"FileByteCounter"`
-	MaximumFileKey      uint64 `json:"MaxFileKey"`
-}
-
-func appendToIndexFile(indexFile *os.File,
-	key uint64, offset uint32, size uint32) error {
-	bytes := make([]byte, 16)
-	util.Uint64toBytes(bytes[0:8], key)
-	util.Uint32toBytes(bytes[8:12], offset)
-	util.Uint32toBytes(bytes[12:16], size)
-	if _, err := indexFile.Seek(0, 2); err != nil {
-		return fmt.Errorf("cannot seek end of indexfile %s: %v",
-			indexFile.Name(), err)
-	}
-	_, err := indexFile.Write(bytes)
-	return err
-}
-
-func (mm *mapMetric) logDelete(deletedByteCount uint32) {
-	mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(deletedByteCount)
-	mm.DeletionCounter++
-}
-
-func (mm *mapMetric) logPut(key uint64, oldSize uint32, newSize uint32) {
-	if key > mm.MaximumFileKey {
-		mm.MaximumFileKey = key
-	}
-	mm.FileCounter++
-	mm.FileByteCounter = mm.FileByteCounter + uint64(newSize)
-	if oldSize > 0 {
-		mm.DeletionCounter++
-		mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(oldSize)
-	}
-}

go/storage/volume.go

@@ -1,396 +0,0 @@
-package storage
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"path"
-	"sync"
-	"time"
-
-	"github.com/chrislusf/seaweedfs/go/glog"
-)
-
-type Volume struct {
-	Id            VolumeId
-	dir           string
-	Collection    string
-	dataFile      *os.File
-	nm            NeedleMapper
-	needleMapKind NeedleMapType
-	readOnly      bool
-
-	SuperBlock
-
-	accessLock       sync.Mutex
-	lastModifiedTime uint64 //unix time in seconds
-}
-
-func NewVolume(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL) (v *Volume, e error) {
-	v = &Volume{dir: dirname, Collection: collection, Id: id}
-	v.SuperBlock = SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}
-	v.needleMapKind = needleMapKind
-	e = v.load(true, true, needleMapKind)
-	return
-}
-func (v *Volume) String() string {
-	return fmt.Sprintf("Id:%v, dir:%s, Collection:%s, dataFile:%v, nm:%v, readOnly:%v", v.Id, v.dir, v.Collection, v.dataFile, v.nm, v.readOnly)
-}
-
-func loadVolumeWithoutIndex(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType) (v *Volume, e error) {
-	v = &Volume{dir: dirname, Collection: collection, Id: id}
-	v.SuperBlock = SuperBlock{}
-	v.needleMapKind = needleMapKind
-	e = v.load(false, false, needleMapKind)
-	return
-}
-func (v *Volume) FileName() (fileName string) {
-	if v.Collection == "" {
-		fileName = path.Join(v.dir, v.Id.String())
-	} else {
-		fileName = path.Join(v.dir, v.Collection+"_"+v.Id.String())
-	}
-	return
-}
-func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType) error {
-	var e error
-	fileName := v.FileName()
-
-	if exists, canRead, canWrite, modifiedTime := checkFile(fileName + ".dat"); exists {
-		if !canRead {
-			return fmt.Errorf("cannot read Volume Data file %s.dat", fileName)
-		}
-		if canWrite {
-			v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
-			v.lastModifiedTime = uint64(modifiedTime.Unix())
-		} else {
-			glog.V(0).Infoln("opening " + fileName + ".dat in READONLY mode")
-			v.dataFile, e = os.Open(fileName + ".dat")
-			v.readOnly = true
-		}
-	} else {
-		if createDatIfMissing {
-			v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
-		} else {
-			return fmt.Errorf("Volume Data file %s.dat does not exist.", fileName)
-		}
-	}
-
-	if e != nil {
-		if !os.IsPermission(e) {
-			return fmt.Errorf("cannot load Volume Data %s.dat: %v", fileName, e)
-		}
-	}
-
-	if v.ReplicaPlacement == nil {
-		e = v.readSuperBlock()
-	} else {
-		e = v.maybeWriteSuperBlock()
-	}
-	if e == nil && alsoLoadIndex {
-		var indexFile *os.File
-		if v.readOnly {
-			glog.V(1).Infoln("open to read file", fileName+".idx")
-			if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); e != nil {
-				return fmt.Errorf("cannot read Volume Index %s.idx: %v", fileName, e)
-			}
-		} else {
-			glog.V(1).Infoln("open to write file", fileName+".idx")
-			if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); e != nil {
-				return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, e)
-			}
-		}
-		switch needleMapKind {
-		case NeedleMapInMemory:
-			glog.V(0).Infoln("loading index file", fileName+".idx", "readonly", v.readOnly)
-			if v.nm, e = LoadNeedleMap(indexFile); e != nil {
-				glog.V(0).Infof("loading index %s error: %v", fileName+".idx", e)
-			}
-		case NeedleMapLevelDb:
-			glog.V(0).Infoln("loading leveldb file", fileName+".ldb")
-			if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile); e != nil {
-				glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
-			}
-		case NeedleMapBoltDb:
-			glog.V(0).Infoln("loading boltdb file", fileName+".bdb")
-			if v.nm, e = NewBoltDbNeedleMap(fileName+".bdb", indexFile); e != nil {
-				glog.V(0).Infof("loading boltdb %s error: %v", fileName+".bdb", e)
-			}
-		}
-	}
-	return e
-}
-func (v *Volume) Version() Version {
-	return v.SuperBlock.Version()
-}
-func (v *Volume) Size() int64 {
-	stat, e := v.dataFile.Stat()
-	if e == nil {
-		return stat.Size()
-	}
-	glog.V(0).Infof("Failed to read file size %s %v", v.dataFile.Name(), e)
-	return -1
-}
-
-// Close cleanly shuts down this volume
-func (v *Volume) Close() {
-	v.accessLock.Lock()
-	defer v.accessLock.Unlock()
-	v.nm.Close()
-	_ = v.dataFile.Close()
-}
-
-func (v *Volume) NeedToReplicate() bool {
-	return v.ReplicaPlacement.GetCopyCount() > 1
-}
-
-// isFileUnchanged checks whether this needle to write is same as last one.
-// It requires serialized access in the same volume.
-func (v *Volume) isFileUnchanged(n *Needle) bool {
-	nv, ok := v.nm.Get(n.Id)
-	if ok && nv.Offset > 0 {
-		oldNeedle := new(Needle)
-		_, err := oldNeedle.Read(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
-		if err != nil {
-			glog.V(0).Infof("Failed to check updated file %v", err)
-			return false
-		}
-		if oldNeedle.Checksum == n.Checksum && bytes.Equal(oldNeedle.Data, n.Data) {
-			n.DataSize = oldNeedle.DataSize
-			return true
-		}
-	}
-	return false
-}
-
-// Destroy removes everything related to this volume
-func (v *Volume) Destroy() (err error) {
-	if v.readOnly {
-		err = fmt.Errorf("%s is read-only", v.dataFile.Name())
-		return
-	}
-	v.Close()
-	err = os.Remove(v.dataFile.Name())
-	if err != nil {
-		return
-	}
-	err = v.nm.Destroy()
-	return
-}
-
-func (v *Volume) write(n *Needle) (size uint32, err error) {
-	glog.V(4).Infof("writing needle %s", NewFileIdFromNeedle(v.Id, n).String())
-	if v.readOnly {
-		err = fmt.Errorf("%s is read-only", v.dataFile.Name())
-		return
-	}
-	v.accessLock.Lock()
-	defer v.accessLock.Unlock()
-	if v.isFileUnchanged(n) {
-		size = n.DataSize
-		glog.V(4).Infof("needle is unchanged!")
-		return
-	}
-	var offset int64
-	if offset, err = v.dataFile.Seek(0, 2); err != nil {
-		glog.V(0).Infof("failed to seek the end of file: %v", err)
-		return
-	}
-
-	//ensure file writing starting from aligned positions
-	if offset%NeedlePaddingSize != 0 {
-		offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize)
-		if offset, err = v.dataFile.Seek(offset, 0); err != nil {
-			glog.V(0).Infof("failed to align in datafile %s: %v", v.dataFile.Name(), err)
-			return
-		}
-	}
-
-	if size, err = n.Append(v.dataFile, v.Version()); err != nil {
-		if e := v.dataFile.Truncate(offset); e != nil {
-			err = fmt.Errorf("%s\ncannot truncate %s: %v", err, v.dataFile.Name(), e)
-		}
-		return
-	}
-	nv, ok := v.nm.Get(n.Id)
-	if !ok || int64(nv.Offset)*NeedlePaddingSize < offset {
-		if err = v.nm.Put(n.Id, uint32(offset/NeedlePaddingSize), n.Size); err != nil {
-			glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err)
-		}
-	}
-	if v.lastModifiedTime < n.LastModified {
-		v.lastModifiedTime = n.LastModified
-	}
-	return
-}
-
-func (v *Volume) delete(n *Needle) (uint32, error) {
-	glog.V(4).Infof("delete needle %s", NewFileIdFromNeedle(v.Id, n).String())
-	if v.readOnly {
-		return 0, fmt.Errorf("%s is read-only", v.dataFile.Name())
-	}
-	v.accessLock.Lock()
-	defer v.accessLock.Unlock()
-	nv, ok := v.nm.Get(n.Id)
-	//fmt.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
-	if ok {
-		size := nv.Size
-		if err := v.nm.Delete(n.Id); err != nil {
-			return size, err
-		}
-		if _, err := v.dataFile.Seek(0, 2); err != nil {
-			return size, err
-		}
-		n.Data = nil
-		_, err := n.Append(v.dataFile, v.Version())
-		return size, err
-	}
-	return 0, nil
-}
-
-func (v *Volume) read(n *Needle) (int, error) {
-	nv, ok := v.nm.Get(n.Id)
-	if !ok || nv.Offset == 0 {
-		return -1, errors.New("Not Found")
-	}
-	bytesRead, err := n.Read(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
-	if err != nil {
-		return bytesRead, err
-	}
-	if !n.HasTtl() {
-		return bytesRead, err
-	}
-	ttlMinutes := n.Ttl.Minutes()
-	if ttlMinutes == 0 {
-		return bytesRead, nil
-	}
-	if !n.HasLastModifiedDate() {
-		return bytesRead, nil
-	}
-	if uint64(time.Now().Unix()) < n.LastModified+uint64(ttlMinutes*60) {
-		return bytesRead, nil
-	}
-	return -1, errors.New("Not Found")
-}
-
-func ScanVolumeFile(dirname string, collection string, id VolumeId,
-	needleMapKind NeedleMapType,
-	visitSuperBlock func(SuperBlock) error,
-	readNeedleBody bool,
-	visitNeedle func(n *Needle, offset int64) error) (err error) {
-	var v *Volume
-	if v, err = loadVolumeWithoutIndex(dirname, collection, id, needleMapKind); err != nil {
-		return fmt.Errorf("Failed to load volume %d: %v", id, err)
-	}
-	if err = visitSuperBlock(v.SuperBlock); err != nil {
-		return fmt.Errorf("Failed to read volume %d super block: %v", id, err)
-	}
-
-	version := v.Version()
-
-	offset := int64(SuperBlockSize)
-	n, rest, e := ReadNeedleHeader(v.dataFile, version, offset)
-	if e != nil {
-		err = fmt.Errorf("cannot read needle header: %v", e)
-		return
-	}
-	for n != nil {
-		if readNeedleBody {
-			if err = n.ReadNeedleBody(v.dataFile, version, offset+int64(NeedleHeaderSize), rest); err != nil {
-				glog.V(0).Infof("cannot read needle body: %v", err)
-				//err = fmt.Errorf("cannot read needle body: %v", err)
-				//return
-			}
-			if n.DataSize >= n.Size {
-				// this should come from a bug reported on #87 and #93
-				// fixed in v0.69
-				// remove this whole "if" clause later, long after 0.69
-				oldRest, oldSize := rest, n.Size
-				padding := NeedlePaddingSize - ((n.Size + NeedleHeaderSize + NeedleChecksumSize) % NeedlePaddingSize)
-				n.Size = 0
-				rest = n.Size + NeedleChecksumSize + padding
-				if rest%NeedlePaddingSize != 0 {
-					rest += (NeedlePaddingSize - rest%NeedlePaddingSize)
-				}
-				glog.V(4).Infof("Adjusting n.Size %d=>0 rest:%d=>%d %+v", oldSize, oldRest, rest, n)
-			}
-		}
-		if err = visitNeedle(n, offset); err != nil {
-			glog.V(0).Infof("visit needle error: %v", err)
-		}
-		offset += int64(NeedleHeaderSize) + int64(rest)
-		glog.V(4).Infof("==> new entry offset %d", offset)
-		if n, rest, err = ReadNeedleHeader(v.dataFile, version, offset); err != nil {
-			if err == io.EOF {
-				return nil
-			}
-			return fmt.Errorf("cannot read needle header: %v", err)
-		}
-		glog.V(4).Infof("new entry needle size:%d rest:%d", n.Size, rest)
-	}
-
-	return
-}
-
-func (v *Volume) ContentSize() uint64 {
-	return v.nm.ContentSize()
-}
-
-func checkFile(filename string) (exists, canRead, canWrite bool, modTime time.Time) {
-	exists = true
-	fi, err := os.Stat(filename)
-	if os.IsNotExist(err) {
-		exists = false
-		return
-	}
-	if fi.Mode()&0400 != 0 {
-		canRead = true
-	}
-	if fi.Mode()&0200 != 0 {
-		canWrite = true
-	}
-	modTime = fi.ModTime()
-	return
-}
-
-// volume is expired if modified time + volume ttl < now
-// except when volume is empty
-// or when the volume does not have a ttl
-// or when volumeSizeLimit is 0 when server just starts
-func (v *Volume) expired(volumeSizeLimit uint64) bool {
-	if volumeSizeLimit == 0 {
-		//skip if we don't know size limit
-		return false
-	}
-	if v.ContentSize() == 0 {
-		return false
-	}
-	if v.Ttl == nil || v.Ttl.Minutes() == 0 {
-		return false
-	}
-	glog.V(0).Infof("now:%v lastModified:%v", time.Now().Unix(), v.lastModifiedTime)
-	livedMinutes := (time.Now().Unix() - int64(v.lastModifiedTime)) / 60
-	glog.V(0).Infof("ttl:%v lived:%v", v.Ttl, livedMinutes)
-	if int64(v.Ttl.Minutes()) < livedMinutes {
-		return true
-	}
-	return false
-}
-
-// wait either maxDelayMinutes or 10% of ttl minutes
-func (v *Volume) exiredLongEnough(maxDelayMinutes uint32) bool {
-	if v.Ttl == nil || v.Ttl.Minutes() == 0 {
-		return false
-	}
-	removalDelay := v.Ttl.Minutes() / 10
-	if removalDelay > maxDelayMinutes {
-		removalDelay = maxDelayMinutes
-	}
-
-	if uint64(v.Ttl.Minutes()+removalDelay)*60+v.lastModifiedTime < uint64(time.Now().Unix()) {
-		return true
-	}
-	return false
-}

go/storage/volume_info.go

@@ -1,44 +0,0 @@
-package storage
-
-import (
-	"fmt"
-	"github.com/chrislusf/seaweedfs/go/operation"
-)
-
-type VolumeInfo struct {
-	Id               VolumeId
-	Size             uint64
-	ReplicaPlacement *ReplicaPlacement
-	Ttl              *TTL
-	Collection       string
-	Version          Version
-	FileCount        int
-	DeleteCount      int
-	DeletedByteCount uint64
-	ReadOnly         bool
-}
-
-func NewVolumeInfo(m *operation.VolumeInformationMessage) (vi VolumeInfo, err error) {
-	vi = VolumeInfo{
-		Id:               VolumeId(*m.Id),
-		Size:             *m.Size,
-		Collection:       *m.Collection,
-		FileCount:        int(*m.FileCount),
-		DeleteCount:      int(*m.DeleteCount),
-		DeletedByteCount: *m.DeletedByteCount,
-		ReadOnly:         *m.ReadOnly,
-		Version:          Version(*m.Version),
-	}
-	rp, e := NewReplicaPlacementFromByte(byte(*m.ReplicaPlacement))
-	if e != nil {
-		return vi, e
-	}
-	vi.ReplicaPlacement = rp
-	vi.Ttl = LoadTTLFromUint32(*m.Ttl)
-	return vi, nil
-}
-
-func (vi VolumeInfo) String() string {
-	return fmt.Sprintf("Id:%d, Size:%d, ReplicaPlacement:%s, Collection:%s, Version:%v, FileCount:%d, DeleteCount:%d, DeletedByteCount:%d, ReadOnly:%v",
-		vi.Id, vi.Size, vi.ReplicaPlacement, vi.Collection, vi.Version, vi.FileCount, vi.DeleteCount, vi.DeletedByteCount, vi.ReadOnly)
-}

go/storage/volume_vacuum.go

@@ -1,90 +0,0 @@
-package storage
-
-import (
-	"fmt"
-	"os"
-	"time"
-
-	"github.com/chrislusf/seaweedfs/go/glog"
-)
-
-func (v *Volume) garbageLevel() float64 {
-	return float64(v.nm.DeletedSize()) / float64(v.ContentSize())
-}
-
-func (v *Volume) Compact() error {
-	glog.V(3).Infof("Compacting ...")
-	//no need to lock for copy on write
-	//v.accessLock.Lock()
-	//defer v.accessLock.Unlock()
-	//glog.V(3).Infof("Got Compaction lock...")
-
-	filePath := v.FileName()
-	glog.V(3).Infof("creating copies for volume %d ...", v.Id)
-	return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx")
-}
-func (v *Volume) commitCompact() error {
-	glog.V(3).Infof("Committing vacuuming...")
-	v.accessLock.Lock()
-	defer v.accessLock.Unlock()
-	glog.V(3).Infof("Got Committing lock...")
-	_ = v.dataFile.Close()
-	var e error
-	if e = os.Rename(v.FileName()+".cpd", v.FileName()+".dat"); e != nil {
-		return e
-	}
-	if e = os.Rename(v.FileName()+".cpx", v.FileName()+".idx"); e != nil {
-		return e
-	}
-	//glog.V(3).Infof("Pretending to be vacuuming...")
-	//time.Sleep(20 * time.Second)
-	if e = v.load(true, false, v.needleMapKind); e != nil {
-		return e
-	}
-	return nil
-}
-
-func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) {
-	var (
-		dst, idx *os.File
-	)
-	if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
-		return
-	}
-	defer dst.Close()
-
-	if idx, err = os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
-		return
-	}
-	defer idx.Close()
-
-	nm := NewNeedleMap(idx)
-	new_offset := int64(SuperBlockSize)
-
-	now := uint64(time.Now().Unix())
-
-	err = ScanVolumeFile(v.dir, v.Collection, v.Id, v.needleMapKind,
-		func(superBlock SuperBlock) error {
-			_, err = dst.Write(superBlock.Bytes())
-			return err
-		}, true, func(n *Needle, offset int64) error {
-			if n.HasTtl() && now >= n.LastModified+uint64(v.Ttl.Minutes()*60) {
-				return nil
-			}
-			nv, ok := v.nm.Get(n.Id)
-			glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv)
-			if ok && int64(nv.Offset)*NeedlePaddingSize == offset && nv.Size > 0 {
-				if err = nm.Put(n.Id, uint32(new_offset/NeedlePaddingSize), n.Size); err != nil {
-					return fmt.Errorf("cannot put needle: %s", err)
-				}
-				if _, err = n.Append(dst, v.Version()); err != nil {
-					return fmt.Errorf("cannot append needle: %s", err)
-				}
-				new_offset += n.DiskSize()
-				glog.V(3).Infoln("saving key", n.Id, "volume offset", offset, "=>", new_offset, "data_size", n.Size)
-			}
-			return nil
-		})
-
-	return
-}

go/topology/store_replicate.go

@@ -1,103 +0,0 @@
-package topology
-
-import (
-	"bytes"
-	"net/http"
-	"strconv"
-
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/operation"
-	"github.com/chrislusf/seaweedfs/go/security"
-	"github.com/chrislusf/seaweedfs/go/storage"
-	"github.com/chrislusf/seaweedfs/go/util"
-)
-
-func ReplicatedWrite(masterNode string, s *storage.Store,
-	volumeId storage.VolumeId, needle *storage.Needle,
-	r *http.Request) (size uint32, errorStatus string) {
-
-	//check JWT
-	jwt := security.GetJwt(r)
-
-	ret, err := s.Write(volumeId, needle)
-	needToReplicate := !s.HasVolume(volumeId)
-	if err != nil {
-		errorStatus = "Failed to write to local disk (" + err.Error() + ")"
-	} else if ret > 0 {
-		needToReplicate = needToReplicate || s.GetVolume(volumeId).NeedToReplicate()
-	} else {
-		errorStatus = "Failed to write to local disk"
-	}
-	if !needToReplicate && ret > 0 {
-		needToReplicate = s.GetVolume(volumeId).NeedToReplicate()
-	}
-	if needToReplicate { //send to other replica locations
-		if r.FormValue("type") != "replicate" {
-			if !distributedOperation(masterNode, s, volumeId, func(location operation.Location) bool {
-				_, err := operation.Upload(
-					"http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified, 10),
-					string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime),
-					jwt)
-				return err == nil
-			}) {
-				ret = 0
-				errorStatus = "Failed to write to replicas for volume " + volumeId.String()
-			}
-		}
-	}
-	size = ret
-	return
-}
-
-func ReplicatedDelete(masterNode string, store *storage.Store,
-	volumeId storage.VolumeId, n *storage.Needle,
-	r *http.Request) (ret uint32) {
-
-	//check JWT
-	jwt := security.GetJwt(r)
-
-	ret, err := store.Delete(volumeId, n)
-	if err != nil {
-		glog.V(0).Infoln("delete error:", err)
-		return
-	}
-
-	needToReplicate := !store.HasVolume(volumeId)
-	if !needToReplicate && ret > 0 {
-		needToReplicate = store.GetVolume(volumeId).NeedToReplicate()
-	}
-	if needToReplicate { //send to other replica locations
-		if r.FormValue("type") != "replicate" {
-			if !distributedOperation(masterNode, store, volumeId, func(location operation.Location) bool {
-				return nil == util.Delete("http://"+location.Url+r.URL.Path+"?type=replicate", jwt)
-			}) {
-				ret = 0
-			}
-		}
-	}
-	return
-}
-
-func distributedOperation(masterNode string, store *storage.Store, volumeId storage.VolumeId, op func(location operation.Location) bool) bool {
-	if lookupResult, lookupErr := operation.Lookup(masterNode, volumeId.String()); lookupErr == nil {
-		length := 0
-		selfUrl := (store.Ip + ":" + strconv.Itoa(store.Port))
-		results := make(chan bool)
-		for _, location := range lookupResult.Locations {
-			if location.Url != selfUrl {
-				length++
-				go func(location operation.Location, results chan bool) {
-					results <- op(location)
-				}(location, results)
-			}
-		}
-		ret := true
-		for i := 0; i < length; i++ {
-			ret = ret && <-results
-		}
-		return ret
-	} else {
-		glog.V(0).Infoln("Failed to lookup for", volumeId, lookupErr.Error())
-	}
-	return false
-}

go/util/concurrent_read_map.go

@@ -1,38 +0,0 @@
-package util
-
-import (
-	"sync"
-)
-
-// A mostly for read map, which can thread-safely
-// initialize the map entries.
-type ConcurrentReadMap struct {
-	rmutex sync.RWMutex
-	mutex  sync.Mutex
-	Items  map[string]interface{}
-}
-
-func NewConcurrentReadMap() *ConcurrentReadMap {
-	return &ConcurrentReadMap{Items: make(map[string]interface{})}
-}
-
-func (m *ConcurrentReadMap) initMapEntry(key string, newEntry func() interface{}) (value interface{}) {
-	m.mutex.Lock()
-	defer m.mutex.Unlock()
-	if value, ok := m.Items[key]; ok {
-		return value
-	}
-	value = newEntry()
-	m.Items[key] = value
-	return value
-}
-
-func (m *ConcurrentReadMap) Get(key string, newEntry func() interface{}) interface{} {
-	m.rmutex.RLock()
-	if value, ok := m.Items[key]; ok {
-		m.rmutex.RUnlock()
-		return value
-	}
-	m.rmutex.RUnlock()
-	return m.initMapEntry(key, newEntry)
-}

go/weed/compact.go

@@ -1,46 +0,0 @@
-package main
-
-import (
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/storage"
-)
-
-func init() {
-	cmdCompact.Run = runCompact // break init cycle
-	cmdCompact.IsDebug = cmdCompact.Flag.Bool("debug", false, "enable debug mode")
-}
-
-var cmdCompact = &Command{
-	UsageLine: "compact -dir=/tmp -volumeId=234",
-	Short:     "run weed tool compact on volume file",
-	Long: `Force an compaction to remove deleted files from volume files.
-  The compacted .dat file is stored as .cpd file.
-  The compacted .idx file is stored as .cpx file.
-
-  `,
-}
-
-var (
-	compactVolumePath       = cmdCompact.Flag.String("dir", "/tmp", "data directory to store files")
-	compactVolumeCollection = cmdCompact.Flag.String("collection", "", "volume collection name")
-	compactVolumeId         = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.")
-)
-
-func runCompact(cmd *Command, args []string) bool {
-
-	if *compactVolumeId == -1 {
-		return false
-	}
-
-	vid := storage.VolumeId(*compactVolumeId)
-	v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid,
-		storage.NeedleMapInMemory, nil, nil)
-	if err != nil {
-		glog.Fatalf("Load Volume [ERROR] %s\n", err)
-	}
-	if err = v.Compact(); err != nil {
-		glog.Fatalf("Compact Volume [ERROR] %s\n", err)
-	}
-
-	return true
-}

go/weed/cover.out

@@ -1,477 +0,0 @@
-mode: set
-github.com/chrislusf/seaweedfs/go/weed/weed.go:41.27,43.20 2 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:46.2,46.17 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:43.20,45.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:49.13,56.19 6 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:60.2,60.23 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:71.2,71.31 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:88.2,90.8 3 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:56.19,58.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:60.23,62.32 2 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:68.3,68.9 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:62.32,63.65 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:63.65,66.5 2 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:71.31,72.46 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:72.46,73.28 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:74.4,77.27 4 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:83.4,84.10 2 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:73.28,73.43 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:77.27,82.5 4 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:114.55,118.43 4 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:118.43,120.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:123.34,124.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:127.2,128.43 2 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:124.13,126.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:131.30,133.2 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:135.14,140.2 4 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:143.26,144.20 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:149.2,149.20 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:154.2,156.31 2 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:164.2,165.12 2 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:144.20,148.3 2 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:149.20,152.3 2 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:156.31,157.24 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:157.24,161.4 2 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:170.23,172.2 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:174.13,175.32 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:178.2,178.21 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:175.32,177.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/weed.go:181.35,183.2 1 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:17.13,19.2 1 1
-github.com/chrislusf/seaweedfs/go/weed/master.go:50.50,51.18 1 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:54.2,55.61 2 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:58.2,58.34 1 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:62.2,73.14 6 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:77.2,77.12 1 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:91.2,91.44 1 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:94.2,94.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:51.18,53.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:55.61,57.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:58.34,60.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:73.14,75.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:77.12,80.24 3 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:83.3,84.25 2 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:87.3,88.31 2 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:80.24,82.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:84.25,86.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/master.go:91.44,93.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:11.13,13.2 1 1
-github.com/chrislusf/seaweedfs/go/weed/shell.go:25.53,29.19 4 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:38.2,38.28 1 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:46.2,46.34 1 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:55.2,56.6 2 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:29.19,31.47 2 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:34.3,34.34 1 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:31.47,33.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:34.34,36.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:38.28,40.17 2 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:44.3,44.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:40.17,43.4 2 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:46.34,47.16 1 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:52.3,52.11 1 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:47.16,48.48 1 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:48.48,50.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/shell.go:56.6,60.3 3 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:23.13,34.2 10 1
-github.com/chrislusf/seaweedfs/go/weed/upload.go:59.50,61.37 2 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:102.2,102.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:61.37,62.23 1 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:65.3,65.82 1 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:62.23,64.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:65.82,66.18 1 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:89.4,89.14 1 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:66.18,67.22 1 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:67.22,68.24 1 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:73.6,74.18 2 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:77.6,82.18 4 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:68.24,69.70 1 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:69.70,71.8 1 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:74.18,76.7 1 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:82.18,84.7 1 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:86.5,88.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:91.3,93.15 2 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:96.3,100.29 3 0
-github.com/chrislusf/seaweedfs/go/weed/upload.go:93.15,95.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/signal_handling.go:11.29,22.12 3 0
-github.com/chrislusf/seaweedfs/go/weed/signal_handling.go:22.12,23.28 1 0
-github.com/chrislusf/seaweedfs/go/weed/signal_handling.go:23.28,26.4 2 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:38.13,52.2 13 1
-github.com/chrislusf/seaweedfs/go/weed/volume.go:68.50,69.19 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:72.2,77.44 4 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:84.2,84.46 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:87.2,87.35 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:94.2,94.34 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:98.2,98.17 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:101.2,101.24 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:105.2,105.24 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:108.2,112.27 4 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:116.2,127.14 5 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:130.2,130.27 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:144.2,144.21 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:148.2,148.52 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:151.2,151.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:69.19,71.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:77.44,78.50 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:78.50,80.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:80.4,82.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:84.46,86.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:87.35,88.57 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:88.57,90.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:94.34,96.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:98.17,100.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:101.24,103.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:105.24,107.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:112.27,114.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:127.14,129.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:130.27,134.15 4 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:137.3,137.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:134.15,136.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:137.13,138.66 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:138.66,140.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:144.21,146.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/volume.go:148.52,150.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:20.13,25.2 4 1
-github.com/chrislusf/seaweedfs/go/weed/download.go:41.52,42.27 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:78.2,78.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:42.27,44.15 2 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:48.3,48.21 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:51.3,51.43 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:44.15,46.12 2 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:48.21,50.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:51.43,55.18 4 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:59.4,60.32 2 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:55.18,57.13 2 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:60.32,63.19 3 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:66.5,66.36 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:69.5,69.19 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:63.19,65.6 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:66.36,68.6 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:69.19,71.11 2 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:74.4,76.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:81.91,83.24 2 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:86.2,87.8 2 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:83.24,85.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:90.70,92.16 2 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:95.2,97.33 3 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:100.2,100.12 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:92.16,94.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/download.go:97.33,99.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:12.13,14.2 1 1
-github.com/chrislusf/seaweedfs/go/weed/fix.go:30.47,32.24 1 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:36.2,37.32 2 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:40.2,41.16 2 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:44.2,50.116 5 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:63.2,63.16 1 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:67.2,67.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:32.24,34.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:37.32,39.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:41.16,43.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:50.116,52.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:52.56,54.17 2 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:61.3,61.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:54.17,57.4 2 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:57.4,60.4 2 0
-github.com/chrislusf/seaweedfs/go/weed/fix.go:63.16,65.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:28.13,30.2 1 1
-github.com/chrislusf/seaweedfs/go/weed/server.go:77.13,89.2 11 1
-github.com/chrislusf/seaweedfs/go/weed/server.go:91.50,93.37 2 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:102.2,102.34 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:106.2,108.49 2 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:112.2,112.28 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:116.2,116.23 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:119.2,124.44 5 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:131.2,131.36 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:134.2,134.33 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:140.2,140.29 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:143.2,143.22 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:152.2,152.67 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:156.2,156.34 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:160.2,160.22 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:186.2,192.12 5 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:224.2,226.28 3 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:229.2,232.27 4 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:235.2,247.20 4 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:250.2,250.27 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:264.2,264.21 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:269.2,269.58 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:273.2,273.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:93.37,95.17 2 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:98.3,99.31 2 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:95.17,97.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:102.34,104.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:108.49,110.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:112.28,114.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:116.23,118.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:124.44,125.50 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:125.50,127.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:127.4,129.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:131.36,133.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:134.33,135.57 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:135.57,137.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:140.29,142.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:143.22,144.30 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:148.3,148.68 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:144.30,147.4 2 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:148.68,150.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:152.67,154.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:156.34,158.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:160.22,161.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:161.13,169.22 3 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:172.4,177.16 3 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:180.4,180.51 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:169.22,171.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:177.16,179.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:180.51,182.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:192.12,201.15 5 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:205.3,205.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:218.3,219.51 2 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:201.15,203.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:205.13,210.26 5 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:213.4,215.21 3 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:210.26,212.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:219.51,221.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:226.28,228.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:232.27,234.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:247.20,249.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:250.27,254.15 4 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:257.3,257.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:254.15,256.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:257.13,258.66 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:258.66,260.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:264.21,267.3 2 0
-github.com/chrislusf/seaweedfs/go/weed/server.go:269.58,271.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/filer.go:32.13,46.2 12 1
-github.com/chrislusf/seaweedfs/go/weed/filer.go:70.49,72.56 1 0
-github.com/chrislusf/seaweedfs/go/weed/filer.go:76.2,83.20 3 0
-github.com/chrislusf/seaweedfs/go/weed/filer.go:86.2,91.14 3 0
-github.com/chrislusf/seaweedfs/go/weed/filer.go:94.2,94.49 1 0
-github.com/chrislusf/seaweedfs/go/weed/filer.go:98.2,98.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/filer.go:72.56,74.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/filer.go:83.20,85.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/filer.go:91.14,93.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/filer.go:94.49,96.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:46.13,64.2 17 1
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:104.53,106.19 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:109.2,110.25 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:119.2,119.14 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:123.2,123.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:127.2,127.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:106.19,108.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:110.25,112.17 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:115.3,116.31 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:112.17,114.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:119.14,121.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:123.13,125.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:130.20,136.38 6 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:140.2,143.40 4 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:146.2,154.25 9 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:136.38,139.3 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:143.40,145.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:157.19,165.38 8 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:169.2,175.24 7 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:165.38,168.3 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:183.71,189.25 5 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:207.2,207.25 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:235.2,236.25 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:189.25,191.13 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:191.13,193.38 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:193.38,194.39 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:197.5,198.52 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:194.39,196.6 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:198.52,200.6 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:200.6,202.6 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:207.25,211.93 4 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:211.93,213.61 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:226.4,227.29 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:213.61,214.45 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:220.5,221.30 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:214.45,217.6 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:217.6,219.6 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:222.5,225.5 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:227.29,229.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:230.4,233.4 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:239.53,242.34 3 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:242.34,243.20 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:246.3,246.20 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:249.3,249.28 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:252.3,255.43 4 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:267.3,267.42 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:243.20,244.12 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:246.20,247.12 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:249.28,251.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:255.43,257.47 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:265.4,265.21 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:257.47,258.65 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:258.65,259.32 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:259.32,262.7 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:267.42,269.51 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:269.51,273.5 3 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:273.5,276.5 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:277.4,280.4 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:284.86,286.16 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:289.2,291.6 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:286.16,288.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:291.6,292.10 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:293.3,295.10 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:296.3,298.28 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:303.63,305.16 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:308.2,311.23 3 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:335.2,335.23 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:305.16,307.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:311.23,312.7 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:312.7,313.42 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:313.42,315.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:315.5,316.10 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:319.3,321.7 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:328.3,328.21 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:321.7,322.42 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:322.42,324.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:324.5,325.10 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:328.21,329.41 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:329.41,331.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:361.29,367.2 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:369.44,371.15 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:371.15,373.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:373.3,373.32 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:373.32,375.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:375.3,377.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:380.70,384.6 4 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:384.6,385.10 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:386.3,388.10 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:389.3,391.43 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:396.4,401.72 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:391.43,395.5 3 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:406.30,408.41 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:414.2,424.35 11 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:436.2,437.39 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:446.2,448.35 3 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:454.2,454.39 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:458.2,465.40 7 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:468.2,471.35 4 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:481.2,482.39 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:408.41,413.3 4 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:424.35,427.20 3 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:427.20,428.15 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:431.4,431.15 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:428.15,430.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:431.15,433.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:437.39,439.26 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:442.3,442.26 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:439.26,441.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:442.26,444.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:448.35,449.20 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:449.20,452.4 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:454.39,457.3 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:465.40,467.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:471.35,473.104 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:473.104,476.89 3 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:476.89,478.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:482.39,484.87 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:484.87,487.89 3 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:487.89,489.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:500.56,501.17 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:504.2,504.28 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:509.2,509.12 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:514.2,515.8 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:501.17,503.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:504.28,506.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:506.3,508.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:509.12,510.26 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:510.26,512.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:518.64,521.15 3 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:532.2,532.20 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:521.15,523.24 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:526.3,527.15 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:530.3,530.16 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:523.24,525.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:527.15,529.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:535.46,541.29 2 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:545.2,545.16 1 0
-github.com/chrislusf/seaweedfs/go/weed/benchmark.go:541.29,544.3 2 0
-github.com/chrislusf/seaweedfs/go/weed/command.go:32.33,35.12 3 0
-github.com/chrislusf/seaweedfs/go/weed/command.go:38.2,38.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/command.go:35.12,37.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/command.go:41.27,48.2 6 0
-github.com/chrislusf/seaweedfs/go/weed/command.go:52.35,54.2 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:18.13,20.2 1 1
-github.com/chrislusf/seaweedfs/go/weed/export.go:48.50,50.27 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:54.2,55.17 2 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:84.2,85.29 2 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:88.2,90.16 3 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:93.2,96.16 3 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:100.2,102.116 2 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:120.2,120.16 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:123.2,123.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:50.27,52.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:55.17,56.56 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:61.3,61.68 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:66.3,67.19 2 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:74.3,81.33 5 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:56.56,59.4 2 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:61.68,64.4 2 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:67.19,69.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:69.4,70.46 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:70.46,72.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:85.29,87.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:90.16,92.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:96.16,98.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:102.116,105.3 2 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:105.55,109.56 3 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:118.3,118.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:109.56,111.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:111.4,112.11 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:112.11,114.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:114.5,116.5 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:120.16,122.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:133.91,135.18 2 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:170.2,170.8 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:135.18,143.17 2 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:146.3,148.45 2 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:152.3,153.54 2 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:156.3,156.31 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:143.17,145.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:148.45,150.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:153.54,155.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:157.3,159.34 2 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:162.3,168.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/export.go:159.34,161.4 1 0
-github.com/chrislusf/seaweedfs/go/weed/version.go:17.51,18.20 1 0
-github.com/chrislusf/seaweedfs/go/weed/version.go:22.2,23.13 2 0
-github.com/chrislusf/seaweedfs/go/weed/version.go:18.20,20.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/compact.go:8.13,11.2 2 1
-github.com/chrislusf/seaweedfs/go/weed/compact.go:29.51,31.28 1 0
-github.com/chrislusf/seaweedfs/go/weed/compact.go:35.2,37.16 3 0
-github.com/chrislusf/seaweedfs/go/weed/compact.go:40.2,40.35 1 0
-github.com/chrislusf/seaweedfs/go/weed/compact.go:44.2,44.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/compact.go:31.28,33.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/compact.go:37.16,39.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/compact.go:40.35,42.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount.go:12.13,17.2 4 1
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:19.49,21.29 2 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:26.2,27.16 2 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:32.2,32.21 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:37.2,38.16 2 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:43.2,44.37 2 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:48.2,48.13 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:21.29,24.3 2 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:27.16,30.3 2 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:32.21,35.3 2 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:38.16,40.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:44.37,46.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:56.30,58.2 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:59.58,61.2 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:68.33,70.2 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:72.74,74.14 2 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:77.2,77.33 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:80.2,80.55 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:74.14,76.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:77.33,79.3 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:85.36,87.2 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:89.69,91.79 2 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:97.2,97.78 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:106.2,106.17 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:91.79,92.38 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:92.38,95.4 2 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:97.78,98.33 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:98.33,99.64 1 0
-github.com/chrislusf/seaweedfs/go/weed/mount_std.go:99.64,102.5 2 0

go/weed/download.go

@@ -1,101 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path"
-	"strings"
-
-	"github.com/chrislusf/seaweedfs/go/operation"
-	"github.com/chrislusf/seaweedfs/go/util"
-)
-
-var (
-	downloadReplication *string
-	downloadDir         *string
-)
-
-func init() {
-	cmdDownload.Run = runDownload // break init cycle
-	cmdDownload.IsDebug = cmdDownload.Flag.Bool("debug", false, "verbose debug information")
-	server = cmdDownload.Flag.String("server", "localhost:9333", "SeaweedFS master location")
-	downloadDir = cmdDownload.Flag.String("dir", ".", "Download the whole folder recursively if specified.")
-}
-
-var cmdDownload = &Command{
-	UsageLine: "download -server=localhost:9333 -dir=one_directory fid1 [fid2 fid3 ...]",
-	Short:     "download files by file id",
-	Long: `download files by file id.
-
-  Usually you just need to use curl to lookup the file's volume server, and then download them directly.
-  This download tool combine the two steps into one.
-
-  What's more, if you use "weed upload -maxMB=..." option to upload a big file divided into chunks, you can
-  use this tool to download the chunks and merge them automatically.
-
-  `,
-}
-
-func runDownload(cmd *Command, args []string) bool {
-	for _, fid := range args {
-		filename, content, e := fetchFileId(*server, fid)
-		if e != nil {
-			fmt.Println("Fetch Error:", e)
-			continue
-		}
-		if filename == "" {
-			filename = fid
-		}
-		if strings.HasSuffix(filename, "-list") {
-			filename = filename[0 : len(filename)-len("-list")]
-			fids := strings.Split(string(content), "\n")
-			f, err := os.OpenFile(path.Join(*downloadDir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
-			if err != nil {
-				fmt.Println("File Creation Error:", e)
-				continue
-			}
-			defer f.Close()
-			for _, partId := range fids {
-				var n int
-				_, part, err := fetchFileId(*server, partId)
-				if err == nil {
-					n, err = f.Write(part)
-				}
-				if err == nil && n < len(part) {
-					err = io.ErrShortWrite
-				}
-				if err != nil {
-					fmt.Println("File Write Error:", err)
-					break
-				}
-			}
-		} else {
-			ioutil.WriteFile(path.Join(*downloadDir, filename), content, os.ModePerm)
-		}
-	}
-	return true
-}
-
-func fetchFileId(server string, fileId string) (filename string, content []byte, e error) {
-	fileUrl, lookupError := operation.LookupFileId(server, fileId)
-	if lookupError != nil {
-		return "", nil, lookupError
-	}
-	filename, content, e = util.DownloadUrl(fileUrl)
-	return
-}
-
-func WriteFile(filename string, data []byte, perm os.FileMode) error {
-	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
-	if err != nil {
-		return err
-	}
-	n, err := f.Write(data)
-	f.Close()
-	if err == nil && n < len(data) {
-		err = io.ErrShortWrite
-	}
-	return err
-}

go/weed/export.go

@@ -1,172 +0,0 @@
-package main
-
-import (
-	"archive/tar"
-	"bytes"
-	"fmt"
-	"os"
-	"path"
-	"strconv"
-	"strings"
-	"text/template"
-	"time"
-
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/storage"
-)
-
-func init() {
-	cmdExport.Run = runExport // break init cycle
-}
-
-const (
-	defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}`
-)
-
-var cmdExport = &Command{
-	UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}}",
-	Short:     "list or export files from one volume data file",
-	Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified.
-
-	The format of file name in the tar file can be customized. Default is {{.Mime}}/{{.Id}}:{{.Name}}. Also available is {{.Key}}.
-
-  `,
-}
-
-var (
-	exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "input data directory to store volume data files")
-	exportCollection = cmdExport.Flag.String("collection", "", "the volume collection name")
-	exportVolumeId   = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
-	dest             = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
-	format           = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}")
-	tarFh            *tar.Writer
-	tarHeader        tar.Header
-	fnTmpl           *template.Template
-	fnTmplBuf        = bytes.NewBuffer(nil)
-)
-
-func runExport(cmd *Command, args []string) bool {
-
-	if *exportVolumeId == -1 {
-		return false
-	}
-
-	var err error
-	if *dest != "" {
-		if *dest != "-" && !strings.HasSuffix(*dest, ".tar") {
-			fmt.Println("the output file", *dest, "should be '-' or end with .tar")
-			return false
-		}
-
-		if fnTmpl, err = template.New("name").Parse(*format); err != nil {
-			fmt.Println("cannot parse format " + *format + ": " + err.Error())
-			return false
-		}
-
-		var fh *os.File
-		if *dest == "-" {
-			fh = os.Stdout
-		} else {
-			if fh, err = os.Create(*dest); err != nil {
-				glog.Fatalf("cannot open output tar %s: %s", *dest, err)
-			}
-		}
-		defer fh.Close()
-		tarFh = tar.NewWriter(fh)
-		defer tarFh.Close()
-		t := time.Now()
-		tarHeader = tar.Header{Mode: 0644,
-			ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
-			Typeflag:   tar.TypeReg,
-			AccessTime: t, ChangeTime: t}
-	}
-
-	fileName := strconv.Itoa(*exportVolumeId)
-	if *exportCollection != "" {
-		fileName = *exportCollection + "_" + fileName
-	}
-	vid := storage.VolumeId(*exportVolumeId)
-	indexFile, err := os.OpenFile(path.Join(*exportVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
-	if err != nil {
-		glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
-	}
-	defer indexFile.Close()
-
-	nm, err := storage.LoadNeedleMap(indexFile)
-	if err != nil {
-		glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
-	}
-
-	var version storage.Version
-
-	err = storage.ScanVolumeFile(*exportVolumePath, *exportCollection, vid,
-		storage.NeedleMapInMemory,
-		func(superBlock storage.SuperBlock) error {
-			version = superBlock.Version()
-			return nil
-		}, true, func(n *storage.Needle, offset int64) error {
-			nv, ok := nm.Get(n.Id)
-			glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
-				n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped(), ok, nv)
-			if ok && nv.Size > 0 && int64(nv.Offset)*8 == offset {
-				return walker(vid, n, version)
-			}
-			if !ok {
-				glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
-			} else {
-				glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
-			}
-			return nil
-		})
-	if err != nil {
-		glog.Fatalf("Export Volume File [ERROR] %s\n", err)
-	}
-	return true
-}
-
-type nameParams struct {
-	Name string
-	Id   uint64
-	Mime string
-	Key  string
-}
-
-func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) {
-	key := storage.NewFileIdFromNeedle(vid, n).String()
-	if tarFh != nil {
-		fnTmplBuf.Reset()
-		if err = fnTmpl.Execute(fnTmplBuf,
-			nameParams{Name: string(n.Name),
-				Id:   n.Id,
-				Mime: string(n.Mime),
-				Key:  key,
-			},
-		); err != nil {
-			return err
-		}
-		nm := fnTmplBuf.String()
-
-		if n.IsGzipped() && path.Ext(nm) != ".gz" {
-			nm = nm + ".gz"
-		}
-
-		tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
-		if err = tarFh.WriteHeader(&tarHeader); err != nil {
-			return err
-		}
-		_, err = tarFh.Write(n.Data)
-	} else {
-		size := n.DataSize
-		if version == storage.Version1 {
-			size = n.Size
-		}
-		fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n",
-			key,
-			n.Name,
-			size,
-			n.IsGzipped(),
-			n.Mime,
-		)
-	}
-	return
-}

go/weed/upload.go

@@ -1,103 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
-
-	"github.com/chrislusf/seaweedfs/go/operation"
-	"github.com/chrislusf/seaweedfs/go/security"
-)
-
-var (
-	uploadReplication *string
-	uploadCollection  *string
-	uploadDir         *string
-	uploadTtl         *string
-	include           *string
-	uploadSecretKey   *string
-	maxMB             *int
-)
-
-func init() {
-	cmdUpload.Run = runUpload // break init cycle
-	cmdUpload.IsDebug = cmdUpload.Flag.Bool("debug", false, "verbose debug information")
-	server = cmdUpload.Flag.String("server", "localhost:9333", "SeaweedFS master location")
-	uploadDir = cmdUpload.Flag.String("dir", "", "Upload the whole folder recursively if specified.")
-	include = cmdUpload.Flag.String("include", "", "pattens of files to upload, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
-	uploadReplication = cmdUpload.Flag.String("replication", "", "replication type")
-	uploadCollection = cmdUpload.Flag.String("collection", "", "optional collection name")
-	uploadTtl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
-	maxMB = cmdUpload.Flag.Int("maxMB", 0, "split files larger than the limit")
-	uploadSecretKey = cmdUpload.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
-}
-
-var cmdUpload = &Command{
-	UsageLine: "upload -server=localhost:9333 file1 [file2 file3]\n upload -server=localhost:9333 -dir=one_directory -include=*.pdf",
-	Short:     "upload one or a list of files",
-	Long: `upload one or a list of files, or batch upload one whole folder recursively.
-
-  If uploading a list of files:
-  It uses consecutive file keys for the list of files.
-  e.g. If the file1 uses key k, file2 can be read via k_1
-
-  If uploading a whole folder recursively:
-  All files under the folder and subfolders will be uploaded, each with its own file key.
-  Optional parameter "-include" allows you to specify the file name patterns.
-
-  If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
-  This can save volume server's gzipped processing and allow customizable gzip compression level.
-  The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
-
-  If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separatedly.
-  The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
-
-  `,
-}
-
-func runUpload(cmd *Command, args []string) bool {
-	secret := security.Secret(*uploadSecretKey)
-	if len(cmdUpload.Flag.Args()) == 0 {
-		if *uploadDir == "" {
-			return false
-		}
-		filepath.Walk(*uploadDir, func(path string, info os.FileInfo, err error) error {
-			if err == nil {
-				if !info.IsDir() {
-					if *include != "" {
-						if ok, _ := filepath.Match(*include, filepath.Base(path)); !ok {
-							return nil
-						}
-					}
-					parts, e := operation.NewFileParts([]string{path})
-					if e != nil {
-						return e
-					}
-					results, e := operation.SubmitFiles(*server, parts,
-						*uploadReplication, *uploadCollection,
-						*uploadTtl, *maxMB, secret)
-					bytes, _ := json.Marshal(results)
-					fmt.Println(string(bytes))
-					if e != nil {
-						return e
-					}
-				}
-			} else {
-				fmt.Println(err)
-			}
-			return err
-		})
-	} else {
-		parts, e := operation.NewFileParts(args)
-		if e != nil {
-			fmt.Println(e.Error())
-		}
-		results, _ := operation.SubmitFiles(*server, parts,
-			*uploadReplication, *uploadCollection,
-			*uploadTtl, *maxMB, secret)
-		bytes, _ := json.Marshal(results)
-		fmt.Println(string(bytes))
-	}
-	return true
-}

go/weed/weed_server/filer_server.go

@@ -1,67 +0,0 @@
-package weed_server
-
-import (
-	"net/http"
-	"strconv"
-
-	"github.com/chrislusf/seaweedfs/go/filer"
-	"github.com/chrislusf/seaweedfs/go/filer/cassandra_store"
-	"github.com/chrislusf/seaweedfs/go/filer/embedded_filer"
-	"github.com/chrislusf/seaweedfs/go/filer/flat_namespace"
-	"github.com/chrislusf/seaweedfs/go/filer/redis_store"
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/security"
-)
-
-type FilerServer struct {
-	port               string
-	master             string
-	collection         string
-	defaultReplication string
-	redirectOnRead     bool
-	disableDirListing  bool
-	secret             security.Secret
-	filer              filer.Filer
-}
-
-func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string,
-	replication string, redirectOnRead bool, disableDirListing bool,
-	secret string,
-	cassandra_server string, cassandra_keyspace string,
-	redis_server string, redis_database int,
-) (fs *FilerServer, err error) {
-	fs = &FilerServer{
-		master:             master,
-		collection:         collection,
-		defaultReplication: replication,
-		redirectOnRead:     redirectOnRead,
-		disableDirListing:  disableDirListing,
-		port:               ":" + strconv.Itoa(port),
-	}
-
-	if cassandra_server != "" {
-		cassandra_store, err := cassandra_store.NewCassandraStore(cassandra_keyspace, cassandra_server)
-		if err != nil {
-			glog.Fatalf("Can not connect to cassandra server %s with keyspace %s: %v", cassandra_server, cassandra_keyspace, err)
-		}
-		fs.filer = flat_namespace.NewFlatNamespaceFiler(master, cassandra_store)
-	} else if redis_server != "" {
-		redis_store := redis_store.NewRedisStore(redis_server, redis_database)
-		fs.filer = flat_namespace.NewFlatNamespaceFiler(master, redis_store)
-	} else {
-		if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil {
-			glog.Fatalf("Can not start filer in dir %s : %v", dir, err)
-			return
-		}
-
-		r.HandleFunc("/admin/mv", fs.moveHandler)
-	}
-
-	r.HandleFunc("/", fs.filerHandler)
-
-	return fs, nil
-}
-
-func (fs *FilerServer) jwt(fileId string) security.EncodedJwt {
-	return security.GenJwt(fs.secret, fileId)
-}

go/weed/weed_server/filer_server_handlers.go

@@ -1,222 +0,0 @@
-package weed_server
-
-import (
-	"encoding/json"
-	"errors"
-	"io"
-	"io/ioutil"
-	"math/rand"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/operation"
-	"github.com/chrislusf/seaweedfs/go/util"
-	"github.com/syndtr/goleveldb/leveldb"
-)
-
-func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
-	switch r.Method {
-	case "GET":
-		fs.GetOrHeadHandler(w, r, true)
-	case "HEAD":
-		fs.GetOrHeadHandler(w, r, false)
-	case "DELETE":
-		fs.DeleteHandler(w, r)
-	case "PUT":
-		fs.PostHandler(w, r)
-	case "POST":
-		fs.PostHandler(w, r)
-	}
-}
-
-// listDirectoryHandler lists directories and folers under a directory
-// files are sorted by name and paginated via "lastFileName" and "limit".
-// sub directories are listed on the first page, when "lastFileName"
-// is empty.
-func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Request) {
-	if !strings.HasSuffix(r.URL.Path, "/") {
-		return
-	}
-	dirlist, err := fs.filer.ListDirectories(r.URL.Path)
-	if err == leveldb.ErrNotFound {
-		glog.V(3).Infoln("Directory Not Found in db", r.URL.Path)
-		w.WriteHeader(http.StatusNotFound)
-		return
-	}
-	m := make(map[string]interface{})
-	m["Directory"] = r.URL.Path
-	lastFileName := r.FormValue("lastFileName")
-	if lastFileName == "" {
-		m["Subdirectories"] = dirlist
-	}
-	limit, limit_err := strconv.Atoi(r.FormValue("limit"))
-	if limit_err != nil {
-		limit = 100
-	}
-	m["Files"], _ = fs.filer.ListFiles(r.URL.Path, lastFileName, limit)
-	writeJsonQuiet(w, r, http.StatusOK, m)
-}
-func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) {
-	if strings.HasSuffix(r.URL.Path, "/") {
-		if fs.disableDirListing {
-			w.WriteHeader(http.StatusMethodNotAllowed)
-			return
-		}
-		fs.listDirectoryHandler(w, r)
-		return
-	}
-	fileId, err := fs.filer.FindFile(r.URL.Path)
-	if err == leveldb.ErrNotFound {
-		glog.V(3).Infoln("Not found in db", r.URL.Path)
-		w.WriteHeader(http.StatusNotFound)
-		return
-	}
-	parts := strings.Split(fileId, ",")
-	if len(parts) != 2 {
-		glog.V(1).Infoln("Invalid fileId", fileId)
-		w.WriteHeader(http.StatusNotFound)
-		return
-	}
-	lookup, lookupError := operation.Lookup(fs.master, parts[0])
-	if lookupError != nil {
-		glog.V(1).Infoln("Invalid lookup", lookupError.Error())
-		w.WriteHeader(http.StatusNotFound)
-		return
-	}
-	if len(lookup.Locations) == 0 {
-		glog.V(1).Infoln("Can not find location for volume", parts[0])
-		w.WriteHeader(http.StatusNotFound)
-		return
-	}
-	urlLocation := lookup.Locations[rand.Intn(len(lookup.Locations))].Url
-	urlString := "http://" + urlLocation + "/" + fileId
-	if fs.redirectOnRead {
-		http.Redirect(w, r, urlString, http.StatusFound)
-		return
-	}
-	u, _ := url.Parse(urlString)
-	request := &http.Request{
-		Method:        r.Method,
-		URL:           u,
-		Proto:         r.Proto,
-		ProtoMajor:    r.ProtoMajor,
-		ProtoMinor:    r.ProtoMinor,
-		Header:        r.Header,
-		Body:          r.Body,
-		Host:          r.Host,
-		ContentLength: r.ContentLength,
-	}
-	glog.V(3).Infoln("retrieving from", u)
-	resp, do_err := util.Do(request)
-	if do_err != nil {
-		glog.V(0).Infoln("failing to connect to volume server", do_err.Error())
-		writeJsonError(w, r, http.StatusInternalServerError, do_err)
-		return
-	}
-	defer resp.Body.Close()
-	for k, v := range resp.Header {
-		w.Header()[k] = v
-	}
-	w.WriteHeader(resp.StatusCode)
-	io.Copy(w, resp.Body)
-}
-
-func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
-	query := r.URL.Query()
-	replication := query.Get("replication")
-	if replication == "" {
-		replication = fs.defaultReplication
-	}
-	assignResult, ae := operation.Assign(fs.master, 1, replication, fs.collection, query.Get("ttl"))
-	if ae != nil {
-		glog.V(0).Infoln("failing to assign a file id", ae.Error())
-		writeJsonError(w, r, http.StatusInternalServerError, ae)
-		return
-	}
-
-	u, _ := url.Parse("http://" + assignResult.Url + "/" + assignResult.Fid)
-	glog.V(4).Infoln("post to", u)
-	request := &http.Request{
-		Method:        r.Method,
-		URL:           u,
-		Proto:         r.Proto,
-		ProtoMajor:    r.ProtoMajor,
-		ProtoMinor:    r.ProtoMinor,
-		Header:        r.Header,
-		Body:          r.Body,
-		Host:          r.Host,
-		ContentLength: r.ContentLength,
-	}
-	resp, do_err := util.Do(request)
-	if do_err != nil {
-		glog.V(0).Infoln("failing to connect to volume server", r.RequestURI, do_err.Error())
-		writeJsonError(w, r, http.StatusInternalServerError, do_err)
-		return
-	}
-	defer resp.Body.Close()
-	resp_body, ra_err := ioutil.ReadAll(resp.Body)
-	if ra_err != nil {
-		glog.V(0).Infoln("failing to upload to volume server", r.RequestURI, ra_err.Error())
-		writeJsonError(w, r, http.StatusInternalServerError, ra_err)
-		return
-	}
-	glog.V(4).Infoln("post result", string(resp_body))
-	var ret operation.UploadResult
-	unmarshal_err := json.Unmarshal(resp_body, &ret)
-	if unmarshal_err != nil {
-		glog.V(0).Infoln("failing to read upload resonse", r.RequestURI, string(resp_body))
-		writeJsonError(w, r, http.StatusInternalServerError, unmarshal_err)
-		return
-	}
-	if ret.Error != "" {
-		glog.V(0).Infoln("failing to post to volume server", r.RequestURI, ret.Error)
-		writeJsonError(w, r, http.StatusInternalServerError, errors.New(ret.Error))
-		return
-	}
-	path := r.URL.Path
-	if strings.HasSuffix(path, "/") {
-		if ret.Name != "" {
-			path += ret.Name
-		} else {
-			operation.DeleteFile(fs.master, assignResult.Fid, fs.jwt(assignResult.Fid)) //clean up
-			glog.V(0).Infoln("Can not to write to folder", path, "without a file name!")
-			writeJsonError(w, r, http.StatusInternalServerError,
-				errors.New("Can not to write to folder "+path+" without a file name"))
-			return
-		}
-	}
-	glog.V(4).Infoln("saving", path, "=>", assignResult.Fid)
-	if db_err := fs.filer.CreateFile(path, assignResult.Fid); db_err != nil {
-		operation.DeleteFile(fs.master, assignResult.Fid, fs.jwt(assignResult.Fid)) //clean up
-		glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err)
-		writeJsonError(w, r, http.StatusInternalServerError, db_err)
-		return
-	}
-	w.WriteHeader(http.StatusCreated)
-	w.Write(resp_body)
-}
-
-// curl -X DELETE http://localhost:8888/path/to
-// curl -X DELETE http://localhost:8888/path/to?recursive=true
-func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
-	var err error
-	var fid string
-	if strings.HasSuffix(r.URL.Path, "/") {
-		isRecursive := r.FormValue("recursive") == "true"
-		err = fs.filer.DeleteDirectory(r.URL.Path, isRecursive)
-	} else {
-		fid, err = fs.filer.DeleteFile(r.URL.Path)
-		if err == nil {
-			err = operation.DeleteFile(fs.master, fid, fs.jwt(fid))
-		}
-	}
-	if err == nil {
-		writeJsonQuiet(w, r, http.StatusAccepted, map[string]string{"error": ""})
-	} else {
-		glog.V(4).Infoln("deleting", r.URL.Path, ":", err.Error())
-		writeJsonError(w, r, http.StatusInternalServerError, err)
-	}
-}

unmaintained/change_superblock/change_superblock.go

@@ -0,0 +1,103 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path"
+	"strconv"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+)
+
+var (
+	fixVolumePath       = flag.String("dir", "/tmp", "data directory to store files")
+	fixVolumeCollection = flag.String("collection", "", "the volume collection name")
+	fixVolumeId         = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
+	targetReplica       = flag.String("replication", "", "If just empty, only print out current replication setting.")
+	targetTTL           = flag.String("ttl", "", "If just empty, only print out current ttl setting.")
+)
+
+/*
+This is to change replication factor in .dat file header. Need to shut down the volume servers
+that has those volumes.
+
+1. fix the .dat file in place
+	// just see the replication setting
+	go run change_replication.go -volumeId=9 -dir=/Users/chrislu/Downloads
+		Current Volume Replication: 000
+	// fix the replication setting
+	go run change_replication.go -volumeId=9 -dir=/Users/chrislu/Downloads -replication 001
+		Current Volume Replication: 000
+		Changing to: 001
+		Done.
+
+2. copy the fixed .dat and related .idx files to some remote server
+3. restart volume servers or start new volume servers.
+*/
+func main() {
+	flag.Parse()
+	fileName := strconv.Itoa(*fixVolumeId)
+	if *fixVolumeCollection != "" {
+		fileName = *fixVolumeCollection + "_" + fileName
+	}
+	datFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".dat"), os.O_RDWR, 0644)
+	if err != nil {
+		glog.Fatalf("Open Volume Data File [ERROR]: %v", err)
+	}
+	defer datFile.Close()
+
+	header := make([]byte, storage.SuperBlockSize)
+	if _, e := datFile.Read(header); e != nil {
+		glog.Fatalf("cannot read volume %s super block: %v", fileName+".dat", e)
+	}
+	superBlock, err := storage.ParseSuperBlock(header)
+
+	if err != nil {
+		glog.Fatalf("cannot parse existing super block: %v", err)
+	}
+
+	fmt.Printf("Current Volume Replication: %s\n", superBlock.ReplicaPlacement)
+	fmt.Printf("Current Volume TTL: %s\n", superBlock.Ttl.String())
+
+	hasChange := false
+
+	if *targetReplica != "" {
+		replica, err := storage.NewReplicaPlacementFromString(*targetReplica)
+
+		if err != nil {
+			glog.Fatalf("cannot parse target replica %s: %v", *targetReplica, err)
+		}
+
+		fmt.Printf("Changing replication to: %s\n", replica)
+
+		superBlock.ReplicaPlacement = replica
+		hasChange = true
+	}
+
+	if *targetTTL != "" {
+		ttl, err := storage.ReadTTL(*targetTTL)
+
+		if err != nil {
+			glog.Fatalf("cannot parse target ttl %s: %v", *targetTTL, err)
+		}
+
+		fmt.Printf("Changing ttl to: %s\n", ttl)
+
+		superBlock.Ttl = ttl
+		hasChange = true
+	}
+
+	if hasChange {
+
+		header = superBlock.Bytes()
+
+		if n, e := datFile.WriteAt(header, 0); n == 0 || e != nil {
+			glog.Fatalf("cannot write super block: %v", e)
+		}
+
+		fmt.Println("Change Applied.")
+	}
+
+}

unmaintained/fix_dat/fix_dat.go

@@ -8,9 +8,9 @@ import (
 	"path"
 	"strconv"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/storage"
-	"github.com/chrislusf/seaweedfs/go/util"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 var (
@@ -60,7 +60,7 @@ func main() {
 
 	iterateEntries(datFile, indexFile, func(n *storage.Needle, offset int64) {
 		fmt.Printf("file id=%d name=%s size=%d dataSize=%d\n", n.Id, string(n.Name), n.Size, n.DataSize)
-		s, e := n.Append(newDatFile, storage.Version2)
+		s, _, e := n.Append(newDatFile, storage.Version2)
 		fmt.Printf("size %d error %v\n", s, e)
 	})
 

unmaintained/see_idx/see_idx.go

@@ -6,8 +6,8 @@ import (
 	"path"
 	"strconv"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/storage"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage"
 )
 
 var (

weed/command/backup.go

@@ -0,0 +1,90 @@
+package command
+
+import (
+	"fmt"
+
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+)
+
+var (
+	s BackupOptions
+)
+
+type BackupOptions struct {
+	master     *string
+	collection *string
+	dir        *string
+	volumeId   *int
+}
+
+func init() {
+	cmdBackup.Run = runBackup // break init cycle
+	s.master = cmdBackup.Flag.String("server", "localhost:9333", "SeaweedFS master location")
+	s.collection = cmdBackup.Flag.String("collection", "", "collection name")
+	s.dir = cmdBackup.Flag.String("dir", ".", "directory to store volume data files")
+	s.volumeId = cmdBackup.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
+}
+
+var cmdBackup = &Command{
+	UsageLine: "backup -dir=. -volumeId=234 -server=localhost:9333",
+	Short:     "incrementally backup a volume to local folder",
+	Long: `Incrementally backup volume data.
+	
+	It is expected that you use this inside a script, to loop through
+	all possible volume ids that needs to be backup to local folder.
+	
+	The volume id does not need to exist locally or even remotely.
+	This will help to backup future new volumes.
+	
+	Usually backing up is just copying the .dat (and .idx) files.
+	But it's tricky to incremententally copy the differences.
+	
+	The complexity comes when there are multiple addition, deletion and compaction.
+	This tool will handle them correctly and efficiently, avoiding unnecessary data transporation.
+  `,
+}
+
+func runBackup(cmd *Command, args []string) bool {
+	if *s.volumeId == -1 {
+		return false
+	}
+	vid := storage.VolumeId(*s.volumeId)
+
+	// find volume location, replication, ttl info
+	lookup, err := operation.Lookup(*s.master, vid.String())
+	if err != nil {
+		fmt.Printf("Error looking up volume %d: %v\n", vid, err)
+		return true
+	}
+	volumeServer := lookup.Locations[0].Url
+
+	stats, err := operation.GetVolumeSyncStatus(volumeServer, vid.String())
+	if err != nil {
+		fmt.Printf("Error get volume %d status: %v\n", vid, err)
+		return true
+	}
+	ttl, err := storage.ReadTTL(stats.Ttl)
+	if err != nil {
+		fmt.Printf("Error get volume %d ttl %s: %v\n", vid, stats.Ttl, err)
+		return true
+	}
+	replication, err := storage.NewReplicaPlacementFromString(stats.Replication)
+	if err != nil {
+		fmt.Printf("Error get volume %d replication %s : %v\n", vid, stats.Replication, err)
+		return true
+	}
+
+	v, err := storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0)
+	if err != nil {
+		fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
+		return true
+	}
+
+	if err := v.Synchronize(volumeServer); err != nil {
+		fmt.Printf("Error synchronizing volume %d: %v\n", vid, err)
+		return true
+	}
+
+	return true
+}

weed/command/benchmark.go

@@ -1,4 +1,4 @@
-package main
+package command
 
 import (
 	"bufio"
@@ -14,10 +14,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/operation"
-	"github.com/chrislusf/seaweedfs/go/security"
-	"github.com/chrislusf/seaweedfs/go/util"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 type BenchmarkOptions struct {
@@ -34,8 +34,6 @@ type BenchmarkOptions struct {
 	cpuprofile       *string
 	maxCpu           *int
 	secretKey        *string
-	vid2server       map[string]string //cache for vid locations
-
 }
 
 var (
@@ -59,7 +57,6 @@ func init() {
 	b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
 	b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
 	b.secretKey = cmdBenchmark.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
-	b.vid2server = make(map[string]string)
 	sharedBytes = make([]byte, 1024)
 }
 
@@ -208,7 +205,11 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 		start := time.Now()
 		fileSize := int64(*b.fileSize + rand.Intn(64))
 		fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize}
-		if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil {
+		ar := &operation.VolumeAssignRequest{
+			Count:      1,
+			Collection: *b.collection,
+		}
+		if assignResult, err := operation.Assign(*b.server, ar); err == nil {
 			fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection
 			if _, err := fp.Upload(0, *b.server, secret); err == nil {
 				if rand.Intn(100) < *b.deletePercentage {
@@ -238,7 +239,6 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 
 func readFiles(fileIdLineChan chan string, s *stat) {
 	defer wait.Done()
-	masterLimitChan := make(chan bool, 1)
 	for fid := range fileIdLineChan {
 		if len(fid) == 0 {
 			continue
@@ -252,31 +252,21 @@ func readFiles(fileIdLineChan chan string, s *stat) {
 		parts := strings.SplitN(fid, ",", 2)
 		vid := parts[0]
 		start := time.Now()
-		if server, ok := b.vid2server[vid]; !ok {
-			masterLimitChan <- true
-			if _, now_ok := b.vid2server[vid]; !now_ok {
-				if ret, err := operation.Lookup(*b.server, vid); err == nil {
-					if len(ret.Locations) > 0 {
-						server = ret.Locations[0].Url
-						b.vid2server[vid] = server
-					}
-				}
-			}
-			<-masterLimitChan
-		}
-		if server, ok := b.vid2server[vid]; ok {
-			url := "http://" + server + "/" + fid
-			if bytesRead, err := util.Get(url); err == nil {
-				s.completed++
-				s.transferred += int64(len(bytesRead))
-				readStats.addSample(time.Now().Sub(start))
-			} else {
-				s.failed++
-				fmt.Printf("Failed to read %s error:%v\n", url, err)
-			}
-		} else {
+		ret, err := operation.Lookup(*b.server, vid)
+		if err != nil || len(ret.Locations) == 0 {
 			s.failed++
 			println("!!!! volume id ", vid, " location not found!!!!!")
+			continue
+		}
+		server := ret.Locations[rand.Intn(len(ret.Locations))].Url
+		url := "http://" + server + "/" + fid
+		if bytesRead, err := util.Get(url); err == nil {
+			s.completed++
+			s.transferred += int64(len(bytesRead))
+			readStats.addSample(time.Now().Sub(start))
+		} else {
+			s.failed++
+			fmt.Printf("Failed to read %s error:%v\n", url, err)
 		}
 	}
 }

weed/command/command.go

@@ -1,4 +1,4 @@
-package main
+package command
 
 import (
 	"flag"
@@ -7,6 +7,24 @@ import (
 	"strings"
 )
 
+var Commands = []*Command{
+	cmdBenchmark,
+	cmdBackup,
+	cmdCompact,
+	cmdCopy,
+	cmdFix,
+	cmdServer,
+	cmdMaster,
+	cmdFiler,
+	cmdUpload,
+	cmdDownload,
+	cmdShell,
+	cmdVersion,
+	cmdVolume,
+	cmdExport,
+	cmdMount,
+}
+
 type Command struct {
 	// Run runs the command.
 	// The args are the arguments after the command name.

weed/command/compact.go

@@ -0,0 +1,53 @@
+package command
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+)
+
+func init() {
+	cmdCompact.Run = runCompact // break init cycle
+}
+
+var cmdCompact = &Command{
+	UsageLine: "compact -dir=/tmp -volumeId=234",
+	Short:     "run weed tool compact on volume file",
+	Long: `Force an compaction to remove deleted files from volume files.
+  The compacted .dat file is stored as .cpd file.
+  The compacted .idx file is stored as .cpx file.
+
+  `,
+}
+
+var (
+	compactVolumePath        = cmdCompact.Flag.String("dir", ".", "data directory to store files")
+	compactVolumeCollection  = cmdCompact.Flag.String("collection", "", "volume collection name")
+	compactVolumeId          = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.")
+	compactMethod            = cmdCompact.Flag.Int("method", 0, "option to choose which compact method. use 0 or 1.")
+	compactVolumePreallocate = cmdCompact.Flag.Int64("preallocateMB", 0, "preallocate volume disk space")
+)
+
+func runCompact(cmd *Command, args []string) bool {
+
+	if *compactVolumeId == -1 {
+		return false
+	}
+
+	vid := storage.VolumeId(*compactVolumeId)
+	v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid,
+		storage.NeedleMapInMemory, nil, nil, *compactVolumePreallocate*(1<<20))
+	if err != nil {
+		glog.Fatalf("Load Volume [ERROR] %s\n", err)
+	}
+	if *compactMethod == 0 {
+		if err = v.Compact(); err != nil {
+			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
+		}
+	} else {
+		if err = v.Compact2(); err != nil {
+			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
+		}
+	}
+
+	return true
+}

weed/command/download.go

@@ -0,0 +1,130 @@
+package command
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+var (
+	d DownloadOptions
+)
+
+type DownloadOptions struct {
+	server *string
+	dir    *string
+}
+
+func init() {
+	cmdDownload.Run = runDownload // break init cycle
+	d.server = cmdDownload.Flag.String("server", "localhost:9333", "SeaweedFS master location")
+	d.dir = cmdDownload.Flag.String("dir", ".", "Download the whole folder recursively if specified.")
+}
+
+var cmdDownload = &Command{
+	UsageLine: "download -server=localhost:9333 -dir=one_directory fid1 [fid2 fid3 ...]",
+	Short:     "download files by file id",
+	Long: `download files by file id.
+
+  Usually you just need to use curl to lookup the file's volume server, and then download them directly.
+  This download tool combine the two steps into one.
+
+  What's more, if you use "weed upload -maxMB=..." option to upload a big file divided into chunks, you can
+  use this tool to download the chunks and merge them automatically.
+
+  `,
+}
+
+func runDownload(cmd *Command, args []string) bool {
+	for _, fid := range args {
+		if e := downloadToFile(*d.server, fid, *d.dir); e != nil {
+			fmt.Println("Download Error: ", fid, e)
+		}
+	}
+	return true
+}
+
+func downloadToFile(server, fileId, saveDir string) error {
+	fileUrl, lookupError := operation.LookupFileId(server, fileId)
+	if lookupError != nil {
+		return lookupError
+	}
+	filename, rc, err := util.DownloadUrl(fileUrl)
+	if err != nil {
+		return err
+	}
+	defer rc.Close()
+	if filename == "" {
+		filename = fileId
+	}
+	isFileList := false
+	if strings.HasSuffix(filename, "-list") {
+		// old command compatible
+		isFileList = true
+		filename = filename[0 : len(filename)-len("-list")]
+	}
+	f, err := os.OpenFile(path.Join(saveDir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if isFileList {
+		content, err := ioutil.ReadAll(rc)
+		if err != nil {
+			return err
+		}
+		fids := strings.Split(string(content), "\n")
+		for _, partId := range fids {
+			var n int
+			_, part, err := fetchContent(*d.server, partId)
+			if err == nil {
+				n, err = f.Write(part)
+			}
+			if err == nil && n < len(part) {
+				err = io.ErrShortWrite
+			}
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		if _, err = io.Copy(f, rc); err != nil {
+			return err
+		}
+
+	}
+	return nil
+}
+
+func fetchContent(server string, fileId string) (filename string, content []byte, e error) {
+	fileUrl, lookupError := operation.LookupFileId(server, fileId)
+	if lookupError != nil {
+		return "", nil, lookupError
+	}
+	var rc io.ReadCloser
+	if filename, rc, e = util.DownloadUrl(fileUrl); e != nil {
+		return "", nil, e
+	}
+	content, e = ioutil.ReadAll(rc)
+	rc.Close()
+	return
+}
+
+func WriteFile(filename string, data []byte, perm os.FileMode) error {
+	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
+	if err != nil {
+		return err
+	}
+	n, err := f.Write(data)
+	f.Close()
+	if err == nil && n < len(data) {
+		err = io.ErrShortWrite
+	}
+	return err
+}

weed/command/export.go

@@ -0,0 +1,213 @@
+package command
+
+import (
+	"archive/tar"
+	"bytes"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+)
+
+const (
+	defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}`
+	timeFormat      = "2006-01-02T15:04:05"
+)
+
+var (
+	export ExportOptions
+)
+
+type ExportOptions struct {
+	dir        *string
+	collection *string
+	volumeId   *int
+}
+
+var cmdExport = &Command{
+	UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}} -newer='" + timeFormat + "'",
+	Short:     "list or export files from one volume data file",
+	Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified.
+
+	The format of file name in the tar file can be customized. Default is {{.Mime}}/{{.Id}}:{{.Name}}. Also available is {{.Key}}.
+
+  `,
+}
+
+func init() {
+	cmdExport.Run = runExport // break init cycle
+	export.dir = cmdExport.Flag.String("dir", ".", "input data directory to store volume data files")
+	export.collection = cmdExport.Flag.String("collection", "", "the volume collection name")
+	export.volumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
+}
+
+var (
+	output = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
+	format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename formatted with {{.Mime}} {{.Id}} {{.Name}} {{.Ext}}")
+	newer  = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone, e.g. 2006-01-02T15:04:05")
+
+	tarOutputFile          *tar.Writer
+	tarHeader              tar.Header
+	fileNameTemplate       *template.Template
+	fileNameTemplateBuffer = bytes.NewBuffer(nil)
+	newerThan              time.Time
+	newerThanUnix          int64 = -1
+	localLocation, _             = time.LoadLocation("Local")
+)
+
+func runExport(cmd *Command, args []string) bool {
+
+	var err error
+
+	if *newer != "" {
+		if newerThan, err = time.ParseInLocation(timeFormat, *newer, localLocation); err != nil {
+			fmt.Println("cannot parse 'newer' argument: " + err.Error())
+			return false
+		}
+		newerThanUnix = newerThan.Unix()
+	}
+
+	if *export.volumeId == -1 {
+		return false
+	}
+
+	if *output != "" {
+		if *output != "-" && !strings.HasSuffix(*output, ".tar") {
+			fmt.Println("the output file", *output, "should be '-' or end with .tar")
+			return false
+		}
+
+		if fileNameTemplate, err = template.New("name").Parse(*format); err != nil {
+			fmt.Println("cannot parse format " + *format + ": " + err.Error())
+			return false
+		}
+
+		var outputFile *os.File
+		if *output == "-" {
+			outputFile = os.Stdout
+		} else {
+			if outputFile, err = os.Create(*output); err != nil {
+				glog.Fatalf("cannot open output tar %s: %s", *output, err)
+			}
+		}
+		defer outputFile.Close()
+		tarOutputFile = tar.NewWriter(outputFile)
+		defer tarOutputFile.Close()
+		t := time.Now()
+		tarHeader = tar.Header{Mode: 0644,
+			ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
+			Typeflag:   tar.TypeReg,
+			AccessTime: t, ChangeTime: t}
+	}
+
+	fileName := strconv.Itoa(*export.volumeId)
+	if *export.collection != "" {
+		fileName = *export.collection + "_" + fileName
+	}
+	vid := storage.VolumeId(*export.volumeId)
+	indexFile, err := os.OpenFile(path.Join(*export.dir, fileName+".idx"), os.O_RDONLY, 0644)
+	if err != nil {
+		glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
+	}
+	defer indexFile.Close()
+
+	needleMap, err := storage.LoadNeedleMap(indexFile)
+	if err != nil {
+		glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
+	}
+
+	var version storage.Version
+
+	err = storage.ScanVolumeFile(*export.dir, *export.collection, vid,
+		storage.NeedleMapInMemory,
+		func(superBlock storage.SuperBlock) error {
+			version = superBlock.Version()
+			return nil
+		}, true, func(n *storage.Needle, offset int64) error {
+			nv, ok := needleMap.Get(n.Id)
+			glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
+				n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped(), ok, nv)
+			if ok && nv.Size > 0 && int64(nv.Offset)*8 == offset {
+				if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
+					glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
+						n.LastModified, newerThanUnix)
+					return nil
+				}
+				return walker(vid, n, version)
+			}
+			if !ok {
+				glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
+			} else {
+				glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
+			}
+			return nil
+		})
+	if err != nil {
+		glog.Fatalf("Export Volume File [ERROR] %s\n", err)
+	}
+	return true
+}
+
+type nameParams struct {
+	Name string
+	Id   uint64
+	Mime string
+	Key  string
+	Ext  string
+}
+
+func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) {
+	key := storage.NewFileIdFromNeedle(vid, n).String()
+	if tarOutputFile != nil {
+		fileNameTemplateBuffer.Reset()
+		if err = fileNameTemplate.Execute(fileNameTemplateBuffer,
+			nameParams{
+				Name: string(n.Name),
+				Id:   n.Id,
+				Mime: string(n.Mime),
+				Key:  key,
+				Ext:  filepath.Ext(string(n.Name)),
+			},
+		); err != nil {
+			return err
+		}
+
+		fileName := fileNameTemplateBuffer.String()
+
+		if n.IsGzipped() && path.Ext(fileName) != ".gz" {
+			fileName = fileName + ".gz"
+		}
+
+		tarHeader.Name, tarHeader.Size = fileName, int64(len(n.Data))
+		if n.HasLastModifiedDate() {
+			tarHeader.ModTime = time.Unix(int64(n.LastModified), 0)
+		} else {
+			tarHeader.ModTime = time.Unix(0, 0)
+		}
+		tarHeader.ChangeTime = tarHeader.ModTime
+		if err = tarOutputFile.WriteHeader(&tarHeader); err != nil {
+			return err
+		}
+		_, err = tarOutputFile.Write(n.Data)
+	} else {
+		size := n.DataSize
+		if version == storage.Version1 {
+			size = n.Size
+		}
+		fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n",
+			key,
+			n.Name,
+			size,
+			n.IsGzipped(),
+			n.Mime,
+		)
+	}
+	return
+}

weed/command/filer.go

@@ -1,4 +1,4 @@
-package main
+package command
 
 import (
 	"net/http"
@@ -6,9 +6,9 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/util"
-	"github.com/chrislusf/seaweedfs/go/weed/weed_server"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 var (
@@ -17,16 +17,20 @@ var (
 
 type FilerOptions struct {
 	master                  *string
+	ip                      *string
 	port                    *int
 	collection              *string
 	defaultReplicaPlacement *string
 	dir                     *string
 	redirectOnRead          *bool
 	disableDirListing       *bool
+	confFile                *string
+	maxMB                   *int
 	secretKey               *string
 	cassandra_server        *string
 	cassandra_keyspace      *string
 	redis_server            *string
+	redis_password          *string
 	redis_database          *int
 }
 
@@ -34,14 +38,18 @@ func init() {
 	cmdFiler.Run = runFiler // break init cycle
 	f.master = cmdFiler.Flag.String("master", "localhost:9333", "master server location")
 	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection")
+	f.ip = cmdFiler.Flag.String("ip", "", "filer server http listen ip address")
 	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
 	f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data")
 	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
 	f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
+	f.confFile = cmdFiler.Flag.String("confFile", "", "json encoded filer conf file")
+	f.maxMB = cmdFiler.Flag.Int("maxMB", 0, "split files larger than the limit")
 	f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server")
 	f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server")
 	f.redis_server = cmdFiler.Flag.String("redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379")
+	f.redis_password = cmdFiler.Flag.String("redis.password", "", "password in clear text")
 	f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server")
 	f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
 
@@ -76,11 +84,13 @@ func runFiler(cmd *Command, args []string) bool {
 	}
 
 	r := http.NewServeMux()
-	_, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection,
+	_, nfs_err := weed_server.NewFilerServer(r, *f.ip, *f.port, *f.master, *f.dir, *f.collection,
 		*f.defaultReplicaPlacement, *f.redirectOnRead, *f.disableDirListing,
+		*f.confFile,
+		*f.maxMB,
 		*f.secretKey,
 		*f.cassandra_server, *f.cassandra_keyspace,
-		*f.redis_server, *f.redis_database,
+		*f.redis_server, *f.redis_password, *f.redis_database,
 	)
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)

weed/command/filer_copy.go

@@ -0,0 +1,147 @@
+package command
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	filer_operation "github.com/chrislusf/seaweedfs/weed/operation/filer"
+	"github.com/chrislusf/seaweedfs/weed/security"
+)
+
+var (
+	copy CopyOptions
+)
+
+type CopyOptions struct {
+	master      *string
+	include     *string
+	replication *string
+	collection  *string
+	ttl         *string
+	maxMB       *int
+	secretKey   *string
+
+	secret security.Secret
+}
+
+func init() {
+	cmdCopy.Run = runCopy // break init cycle
+	cmdCopy.IsDebug = cmdCopy.Flag.Bool("debug", false, "verbose debug information")
+	copy.master = cmdCopy.Flag.String("master", "localhost:9333", "SeaweedFS master location")
+	copy.include = cmdCopy.Flag.String("include", "", "pattens of files to copy, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
+	copy.replication = cmdCopy.Flag.String("replication", "", "replication type")
+	copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
+	copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
+	copy.maxMB = cmdCopy.Flag.Int("maxMB", 0, "split files larger than the limit")
+	copy.secretKey = cmdCopy.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+}
+
+var cmdCopy = &Command{
+	UsageLine: "filer.copy file_or_dir1 [file_or_dir2 file_or_dir3] http://localhost:8888/path/to/a/folder/",
+	Short:     "copy one or a list of files to a filer folder",
+	Long: `copy one or a list of files, or batch copy one whole folder recursively, to a filer folder
+
+  It can copy one or a list of files or folders.
+
+  If copying a whole folder recursively:
+  All files under the folder and subfolders will be copyed.
+  Optional parameter "-include" allows you to specify the file name patterns.
+
+  If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
+  This can save volume server's gzipped processing and allow customizable gzip compression level.
+  The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
+
+  If "maxMB" is set to a positive number, files larger than it would be split into chunks and copyed separatedly.
+  The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
+
+  `,
+}
+
+func runCopy(cmd *Command, args []string) bool {
+	copy.secret = security.Secret(*copy.secretKey)
+	if len(args) <= 1 {
+		return false
+	}
+	filerDestination := args[len(args)-1]
+	fileOrDirs := args[0 : len(args)-1]
+
+	filerUrl, err := url.Parse(filerDestination)
+	if err != nil {
+		fmt.Printf("The last argument should be a URL on filer: %v\n", err)
+		return false
+	}
+	path := filerUrl.Path
+	if !strings.HasSuffix(path, "/") {
+		path = path + "/"
+	}
+
+	for _, fileOrDir := range fileOrDirs {
+		if !doEachCopy(fileOrDir, filerUrl.Host, path) {
+			return false
+		}
+	}
+	return true
+}
+
+func doEachCopy(fileOrDir string, host string, path string) bool {
+	f, err := os.Open(fileOrDir)
+	if err != nil {
+		fmt.Printf("Failed to open file %s: %v", fileOrDir, err)
+		return false
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
+	if err != nil {
+		fmt.Printf("Failed to get stat for file %s: %v", fileOrDir, err)
+		return false
+	}
+
+	mode := fi.Mode()
+	if mode.IsDir() {
+		files, _ := ioutil.ReadDir(fileOrDir)
+		for _, subFileOrDir := range files {
+			if !doEachCopy(fileOrDir+"/"+subFileOrDir.Name(), host, path+fi.Name()+"/") {
+				return false
+			}
+		}
+		return true
+	}
+
+	// this is a regular file
+	if *copy.include != "" {
+		if ok, _ := filepath.Match(*copy.include, filepath.Base(fileOrDir)); !ok {
+			return true
+		}
+	}
+
+	parts, err := operation.NewFileParts([]string{fileOrDir})
+	if err != nil {
+		fmt.Printf("Failed to read file %s: %v", fileOrDir, err)
+	}
+
+	results, err := operation.SubmitFiles(*copy.master, parts,
+		*copy.replication, *copy.collection, "",
+		*copy.ttl, *copy.maxMB, copy.secret)
+	if err != nil {
+		fmt.Printf("Failed to submit file %s: %v", fileOrDir, err)
+	}
+
+	if strings.HasSuffix(path, "/") {
+		path = path + fi.Name()
+	}
+
+	if err = filer_operation.RegisterFile(host, path, results[0].Fid, copy.secret); err != nil {
+		fmt.Printf("Failed to register file %s on %s: %v", fileOrDir, host, err)
+		return false
+	}
+
+	fmt.Printf("Copy %s => http://%s%s\n", fileOrDir, host, path)
+
+	return true
+}

weed/command/fix.go

@@ -1,12 +1,12 @@
-package main
+package command
 
 import (
 	"os"
 	"path"
 	"strconv"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/storage"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage"
 )
 
 func init() {
@@ -22,7 +22,7 @@ var cmdFix = &Command{
 }
 
 var (
-	fixVolumePath       = cmdFix.Flag.String("dir", "/tmp", "data directory to store files")
+	fixVolumePath       = cmdFix.Flag.String("dir", ".", "data directory to store files")
 	fixVolumeCollection = cmdFix.Flag.String("collection", "", "the volume collection name")
 	fixVolumeId         = cmdFix.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
 )
@@ -58,7 +58,7 @@ func runFix(cmd *Command, args []string) bool {
 				glog.V(2).Infof("saved %d with error %v", n.Size, pe)
 			} else {
 				glog.V(2).Infof("skipping deleted file ...")
-				return nm.Delete(n.Id)
+				return nm.Delete(n.Id, uint32(offset/storage.NeedlePaddingSize))
 			}
 			return nil
 		})

weed/command/master.go

@@ -1,17 +1,22 @@
-package main
+package command
 
 import (
 	"net/http"
 	"os"
 	"runtime"
+	"runtime/pprof"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/util"
-	"github.com/chrislusf/seaweedfs/go/weed/weed_server"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
+	"github.com/soheilhy/cmux"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/reflection"
 )
 
 func init() {
@@ -32,16 +37,18 @@ var (
 	masterIp                = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address")
 	masterBindIp            = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
 	metaFolder              = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
-	masterPeers             = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list")
+	masterPeers             = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094")
 	volumeSizeLimitMB       = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
+	volumePreallocate       = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
 	mpulse                  = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
 	confFile                = cmdMaster.Flag.String("conf", "/etc/weedfs/weedfs.conf", "Deprecating! xml configuration file")
 	defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
-	mTimeout                = cmdMaster.Flag.Int("idleTimeout", 10, "connection idle seconds")
-	mMaxCpu                 = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
-	garbageThreshold        = cmdMaster.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
-	masterWhiteListOption   = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
-	masterSecureKey         = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+	// mTimeout                = cmdMaster.Flag.Int("idleTimeout", 30, "connection idle seconds")
+	mMaxCpu               = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
+	garbageThreshold      = cmdMaster.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
+	masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
+	masterSecureKey       = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+	masterCpuProfile      = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")
 
 	masterWhiteList []string
 )
@@ -51,6 +58,17 @@ func runMaster(cmd *Command, args []string) bool {
 		*mMaxCpu = runtime.NumCPU()
 	}
 	runtime.GOMAXPROCS(*mMaxCpu)
+	if *masterCpuProfile != "" {
+		f, err := os.Create(*masterCpuProfile)
+		if err != nil {
+			glog.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+		OnInterrupt(func() {
+			pprof.StopCPUProfile()
+		})
+	}
 	if err := util.TestFolderWritable(*metaFolder); err != nil {
 		glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err)
 	}
@@ -60,7 +78,8 @@ func runMaster(cmd *Command, args []string) bool {
 
 	r := mux.NewRouter()
 	ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
-		*volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold,
+		*volumeSizeLimitMB, *volumePreallocate,
+		*mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold,
 		masterWhiteList, *masterSecureKey,
 	)
 
@@ -68,7 +87,7 @@ func runMaster(cmd *Command, args []string) bool {
 
 	glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)
 
-	listener, e := util.NewListener(listeningAddress, time.Duration(*mTimeout)*time.Second)
+	listener, e := util.NewListener(listeningAddress, 0)
 	if e != nil {
 		glog.Fatalf("Master startup error: %v", e)
 	}
@@ -84,8 +103,25 @@ func runMaster(cmd *Command, args []string) bool {
 		ms.SetRaftServer(raftServer)
 	}()
 
-	if e := http.Serve(listener, r); e != nil {
-		glog.Fatalf("Fail to serve: %v", e)
+	// start grpc and http server
+	m := cmux.New(listener)
+
+	grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
+	httpL := m.Match(cmux.Any())
+
+	// Create your protocol servers.
+	grpcS := grpc.NewServer()
+	pb.RegisterSeaweedServer(grpcS, ms)
+	reflection.Register(grpcS)
+
+	httpS := &http.Server{Handler: r}
+
+	go grpcS.Serve(grpcL)
+	go httpS.Serve(httpL)
+
+	if err := m.Serve(); err != nil {
+		glog.Fatalf("master server failed to serve: %v", err)
 	}
+
 	return true
 }

weed/command/mount.go

@@ -1,4 +1,4 @@
-package main
+package command
 
 type MountOptions struct {
 	filer *string
@@ -13,7 +13,7 @@ func init() {
 	cmdMount.Run = runMount // break init cycle
 	cmdMount.IsDebug = cmdMount.Flag.Bool("debug", false, "verbose debug information")
 	mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location")
-	mountOptions.dir = cmdMount.Flag.String("dir", "", "mount weed filer to this directory")
+	mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
 }
 
 var cmdMount = &Command{

weed/command/mount_notsupported.go

@@ -1,7 +1,7 @@
 // +build !linux
 // +build !darwin
 
-package main
+package command
 
 import (
 	"fmt"

weed/command/mount_std.go

@@ -1,6 +1,6 @@
 // +build linux darwin
 
-package main
+package command
 
 import (
 	"fmt"
@@ -8,10 +8,10 @@ import (
 
 	"bazil.org/fuse"
 	"bazil.org/fuse/fs"
-	"github.com/chrislusf/seaweedfs/go/filer"
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/storage"
-	"github.com/chrislusf/seaweedfs/go/util"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"golang.org/x/net/context"
 )
 
@@ -52,7 +52,8 @@ type File struct {
 	Name   string
 }
 
-func (File) Attr(attr *fuse.Attr) {
+func (File) Attr(context context.Context, attr *fuse.Attr) error {
+	return nil
 }
 func (File) ReadAll(ctx context.Context) ([]byte, error) {
 	return []byte("hello, world\n"), nil
@@ -63,7 +64,8 @@ type Dir struct {
 	Id   uint64
 }
 
-func (dir Dir) Attr(attr *fuse.Attr) {
+func (dir Dir) Attr(context context.Context, attr *fuse.Attr) error {
+	return nil
 }
 
 func (dir Dir) Lookup(ctx context.Context, name string) (fs.Node, error) {

weed/command/server.go

@@ -1,4 +1,4 @@
-package main
+package command
 
 import (
 	"net/http"
@@ -10,11 +10,15 @@ import (
 	"sync"
 	"time"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/storage"
-	"github.com/chrislusf/seaweedfs/go/util"
-	"github.com/chrislusf/seaweedfs/go/weed/weed_server"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
+	"github.com/soheilhy/cmux"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/reflection"
 )
 
 type ServerOptions struct {
@@ -51,7 +55,7 @@ var (
 	serverIp                      = cmdServer.Flag.String("ip", "localhost", "ip or server name")
 	serverBindIp                  = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
 	serverMaxCpu                  = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
-	serverTimeout                 = cmdServer.Flag.Int("idleTimeout", 10, "connection idle seconds")
+	serverTimeout                 = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
 	serverDataCenter              = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
 	serverRack                    = cmdServer.Flag.String("rack", "", "current volume server's rack name")
 	serverWhiteListOption         = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
@@ -61,6 +65,7 @@ var (
 	masterPort                    = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
 	masterMetaFolder              = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
 	masterVolumeSizeLimitMB       = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
+	masterVolumePreallocate       = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
 	masterConfFile                = cmdServer.Flag.String("master.conf", "/etc/weedfs/weedfs.conf", "xml configuration file")
 	masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
 	volumePort                    = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
@@ -70,7 +75,9 @@ var (
 	volumePulse                   = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
 	volumeIndexType               = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb] mode for memory~performance balance.")
 	volumeFixJpgOrientation       = cmdServer.Flag.Bool("volume.images.fix.orientation", true, "Adjust jpg orientation when uploading.")
+	volumeReadRedirect            = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
 	volumeServerPublicUrl         = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
+	volumeEnableBytesCache        = cmdServer.Flag.Bool("volume.cache.enable", false, "direct cache instead of OS cache, cost more memory.")
 	isStartingFiler               = cmdServer.Flag.Bool("filer", false, "whether to start filer")
 
 	serverWhiteList []string
@@ -81,13 +88,16 @@ func init() {
 	filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server")
 	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
 	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
-	filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified")
+	filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -dir is specified")
 	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
 	filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
 	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
+	filerOptions.confFile = cmdServer.Flag.String("filer.confFile", "", "json encoded filer conf file")
+	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 0, "split files larger than the limit")
 	filerOptions.cassandra_server = cmdServer.Flag.String("filer.cassandra.server", "", "host[:port] of the cassandra server")
 	filerOptions.cassandra_keyspace = cmdServer.Flag.String("filer.cassandra.keyspace", "seaweed", "keyspace of the cassandra server")
 	filerOptions.redis_server = cmdServer.Flag.String("filer.redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379")
+	filerOptions.redis_password = cmdServer.Flag.String("filer.redis.password", "", "redis password in clear text")
 	filerOptions.redis_database = cmdServer.Flag.Int("filer.redis.database", 0, "the database on the redis server")
 }
 
@@ -162,13 +172,16 @@ func runServer(cmd *Command, args []string) bool {
 
 	if *isStartingFiler {
 		go func() {
+			time.Sleep(1 * time.Second)
 			r := http.NewServeMux()
-			_, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection,
+			_, nfs_err := weed_server.NewFilerServer(r, *serverBindIp, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection,
 				*filerOptions.defaultReplicaPlacement,
 				*filerOptions.redirectOnRead, *filerOptions.disableDirListing,
+				*filerOptions.confFile,
+				*filerOptions.maxMB,
 				*filerOptions.secretKey,
-				"", "",
-				"", 0,
+				*filerOptions.cassandra_server, *filerOptions.cassandra_keyspace,
+				*filerOptions.redis_server, *filerOptions.redis_password, *filerOptions.redis_database,
 			)
 			if nfs_err != nil {
 				glog.Fatalf("Filer startup error: %v", nfs_err)
@@ -196,12 +209,13 @@ func runServer(cmd *Command, args []string) bool {
 	go func() {
 		r := mux.NewRouter()
 		ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
-			*masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
+			*masterVolumeSizeLimitMB, *masterVolumePreallocate,
+			*volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
 			serverWhiteList, *serverSecureKey,
 		)
 
 		glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort))
-		masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), time.Duration(*serverTimeout)*time.Second)
+		masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), 0)
 		if e != nil {
 			glog.Fatalf("Master startup error: %v", e)
 		}
@@ -220,9 +234,27 @@ func runServer(cmd *Command, args []string) bool {
 		}()
 
 		raftWaitForMaster.Done()
-		if e := http.Serve(masterListener, r); e != nil {
-			glog.Fatalf("Master Fail to serve:%s", e.Error())
+
+		// start grpc and http server
+		m := cmux.New(masterListener)
+
+		grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
+		httpL := m.Match(cmux.Any())
+
+		// Create your protocol servers.
+		grpcS := grpc.NewServer()
+		pb.RegisterSeaweedServer(grpcS, ms)
+		reflection.Register(grpcS)
+
+		httpS := &http.Server{Handler: r}
+
+		go grpcS.Serve(grpcL)
+		go httpS.Serve(httpL)
+
+		if err := m.Serve(); err != nil {
+			glog.Fatalf("master server failed to serve: %v", err)
 		}
+
 	}()
 
 	volumeWait.Wait()
@@ -251,7 +283,8 @@ func runServer(cmd *Command, args []string) bool {
 		folders, maxCounts,
 		volumeNeedleMapKind,
 		*serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack,
-		serverWhiteList, *volumeFixJpgOrientation,
+		serverWhiteList, *volumeFixJpgOrientation, *volumeReadRedirect,
+		*volumeEnableBytesCache,
 	)
 
 	glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort))

weed/command/shell.go

@@ -1,11 +1,11 @@
-package main
+package command
 
 import (
 	"bufio"
 	"fmt"
 	"os"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 )
 
 func init() {

weed/command/signal_handling.go

@@ -1,6 +1,6 @@
 // +build !plan9
 
-package main
+package command
 
 import (
 	"os"
@@ -11,14 +11,17 @@ import (
 func OnInterrupt(fn func()) {
 	// deal with control+c,etc
 	signalChan := make(chan os.Signal, 1)
+	// controlling terminal close, daemon not exit
+	signal.Ignore(syscall.SIGHUP)
 	signal.Notify(signalChan,
 		os.Interrupt,
 		os.Kill,
 		syscall.SIGALRM,
-		syscall.SIGHUP,
+		// syscall.SIGHUP,
 		syscall.SIGINT,
 		syscall.SIGTERM,
-		syscall.SIGQUIT)
+		// syscall.SIGQUIT,
+	)
 	go func() {
 		for _ = range signalChan {
 			fn()

weed/command/signal_handling_notsupported.go

@@ -1,6 +1,6 @@
 // +build plan9
 
-package main
+package command
 
 func OnInterrupt(fn func()) {
 }

weed/command/upload.go

@@ -0,0 +1,110 @@
+package command
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/security"
+)
+
+var (
+	upload UploadOptions
+)
+
+type UploadOptions struct {
+	master      *string
+	dir         *string
+	include     *string
+	replication *string
+	collection  *string
+	dataCenter  *string
+	ttl         *string
+	maxMB       *int
+	secretKey   *string
+}
+
+func init() {
+	cmdUpload.Run = runUpload // break init cycle
+	cmdUpload.IsDebug = cmdUpload.Flag.Bool("debug", false, "verbose debug information")
+	upload.master = cmdUpload.Flag.String("master", "localhost:9333", "SeaweedFS master location")
+	upload.dir = cmdUpload.Flag.String("dir", "", "Upload the whole folder recursively if specified.")
+	upload.include = cmdUpload.Flag.String("include", "", "pattens of files to upload, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
+	upload.replication = cmdUpload.Flag.String("replication", "", "replication type")
+	upload.collection = cmdUpload.Flag.String("collection", "", "optional collection name")
+	upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
+	upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
+	upload.maxMB = cmdUpload.Flag.Int("maxMB", 0, "split files larger than the limit")
+	upload.secretKey = cmdUpload.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+}
+
+var cmdUpload = &Command{
+	UsageLine: "upload -master=localhost:9333 file1 [file2 file3]\n         weed upload -server=localhost:9333 -dir=one_directory -include=*.pdf",
+	Short:     "upload one or a list of files",
+	Long: `upload one or a list of files, or batch upload one whole folder recursively.
+
+  If uploading a list of files:
+  It uses consecutive file keys for the list of files.
+  e.g. If the file1 uses key k, file2 can be read via k_1
+
+  If uploading a whole folder recursively:
+  All files under the folder and subfolders will be uploaded, each with its own file key.
+  Optional parameter "-include" allows you to specify the file name patterns.
+
+  If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
+  This can save volume server's gzipped processing and allow customizable gzip compression level.
+  The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
+
+  If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separatedly.
+  The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
+
+  `,
+}
+
+func runUpload(cmd *Command, args []string) bool {
+	secret := security.Secret(*upload.secretKey)
+	if len(args) == 0 {
+		if *upload.dir == "" {
+			return false
+		}
+		filepath.Walk(*upload.dir, func(path string, info os.FileInfo, err error) error {
+			if err == nil {
+				if !info.IsDir() {
+					if *upload.include != "" {
+						if ok, _ := filepath.Match(*upload.include, filepath.Base(path)); !ok {
+							return nil
+						}
+					}
+					parts, e := operation.NewFileParts([]string{path})
+					if e != nil {
+						return e
+					}
+					results, e := operation.SubmitFiles(*upload.master, parts,
+						*upload.replication, *upload.collection, *upload.dataCenter,
+						*upload.ttl, *upload.maxMB, secret)
+					bytes, _ := json.Marshal(results)
+					fmt.Println(string(bytes))
+					if e != nil {
+						return e
+					}
+				}
+			} else {
+				fmt.Println(err)
+			}
+			return err
+		})
+	} else {
+		parts, e := operation.NewFileParts(args)
+		if e != nil {
+			fmt.Println(e.Error())
+		}
+		results, _ := operation.SubmitFiles(*upload.master, parts,
+			*upload.replication, *upload.collection, *upload.dataCenter,
+			*upload.ttl, *upload.maxMB, secret)
+		bytes, _ := json.Marshal(results)
+		fmt.Println(string(bytes))
+	}
+	return true
+}

weed/command/version.go

@@ -1,10 +1,10 @@
-package main
+package command
 
 import (
 	"fmt"
 	"runtime"
 
-	"github.com/chrislusf/seaweedfs/go/util"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 var cmdVersion = &Command{

weed/command/volume.go

@@ -1,4 +1,4 @@
-package main
+package command
 
 import (
 	"net/http"
@@ -8,10 +8,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/storage"
-	"github.com/chrislusf/seaweedfs/go/util"
-	"github.com/chrislusf/seaweedfs/go/weed/weed_server"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 var (
@@ -35,6 +35,8 @@ type VolumeServerOptions struct {
 	whiteList             []string
 	indexType             *string
 	fixJpgOrientation     *bool
+	readRedirect          *bool
+	enableBytesCache      *bool
 }
 
 func init() {
@@ -46,12 +48,14 @@ func init() {
 	v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
 	v.master = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location")
 	v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
-	v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 10, "connection idle seconds")
+	v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
 	v.maxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
 	v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
 	v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
 	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|boltdb] mode for memory~performance balance.")
 	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", true, "Adjust jpg orientation when uploading.")
+	v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
+	v.enableBytesCache = cmdVolume.Flag.Bool("cache.enable", false, "direct cache instead of OS cache, cost more memory.")
 }
 
 var cmdVolume = &Command{
@@ -129,7 +133,8 @@ func runVolume(cmd *Command, args []string) bool {
 		volumeNeedleMapKind,
 		*v.master, *v.pulseSeconds, *v.dataCenter, *v.rack,
 		v.whiteList,
-		*v.fixJpgOrientation,
+		*v.fixJpgOrientation, *v.readRedirect,
+		*v.enableBytesCache,
 	)
 
 	listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)

weed/command/volume_test.go

@@ -1,11 +1,11 @@
-package main
+package command
 
 import (
 	"net/http"
 	"testing"
 	"time"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 )
 
 func TestXYZ(t *testing.T) {

weed/filer/cassandra_store/cassandra_store.go

@@ -2,8 +2,10 @@ package cassandra_store
 
 import (
 	"fmt"
+	"strings"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 
 	"github.com/gocql/gocql"
 )
@@ -29,9 +31,15 @@ type CassandraStore struct {
 	session *gocql.Session
 }
 
-func NewCassandraStore(keyspace string, hosts ...string) (c *CassandraStore, err error) {
+func NewCassandraStore(keyspace string, hosts string) (c *CassandraStore, err error) {
 	c = &CassandraStore{}
-	c.cluster = gocql.NewCluster(hosts...)
+	s := strings.Split(hosts, ",")
+	if len(s) == 1 {
+		glog.V(2).Info("Only one cassandra node to connect! A cluster is Recommended! Now using:", string(hosts))
+		c.cluster = gocql.NewCluster(hosts)
+	} else if len(s) > 1 {
+		c.cluster = gocql.NewCluster(s...)
+	}
 	c.cluster.Keyspace = keyspace
 	c.cluster.Consistency = gocql.Quorum
 	c.session, err = c.cluster.CreateSession()
@@ -59,6 +67,7 @@ func (c *CassandraStore) Get(fullFileName string) (fid string, err error) {
 		fullFileName).Consistency(gocql.One).Scan(&output); err != nil {
 		if err != gocql.ErrNotFound {
 			glog.V(0).Infof("Failed to find file %s: %v", fullFileName, fid, err)
+			return "", filer.ErrNotFound
 		}
 	}
 	if len(output) == 0 {
@@ -68,16 +77,16 @@ func (c *CassandraStore) Get(fullFileName string) (fid string, err error) {
 }
 
 // Currently the fid is not returned
-func (c *CassandraStore) Delete(fullFileName string) (fid string, err error) {
+func (c *CassandraStore) Delete(fullFileName string) (err error) {
 	if err := c.session.Query(
 		`DELETE FROM seaweed_files WHERE path = ?`,
 		fullFileName).Exec(); err != nil {
 		if err != gocql.ErrNotFound {
 			glog.V(0).Infof("Failed to delete file %s: %v", fullFileName, err)
 		}
-		return "", err
+		return err
 	}
-	return "", nil
+	return nil
 }
 
 func (c *CassandraStore) Close() {

weed/filer/client_operations.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/chrislusf/seaweedfs/go/util"
+	"github.com/chrislusf/seaweedfs/weed/util"
 
 	"net/url"
 )

weed/filer/embedded_filer/directory.go

@@ -1,7 +1,7 @@
 package embedded_filer
 
 import (
-	"github.com/chrislusf/seaweedfs/go/filer"
+	"github.com/chrislusf/seaweedfs/weed/filer"
 )
 
 type DirectoryManager interface {

weed/filer/embedded_filer/directory_in_map.go

@@ -10,19 +10,50 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/chrislusf/seaweedfs/go/filer"
-	"github.com/chrislusf/seaweedfs/go/util"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 var writeLock sync.Mutex //serialize changes to dir.log
 
 type DirectoryEntryInMap struct {
+	sync.Mutex
 	Name           string
 	Parent         *DirectoryEntryInMap
-	SubDirectories map[string]*DirectoryEntryInMap
+	subDirectories map[string]*DirectoryEntryInMap
 	Id             filer.DirectoryId
 }
 
+func (de *DirectoryEntryInMap) getChild(dirName string) (*DirectoryEntryInMap, bool) {
+	de.Lock()
+	defer de.Unlock()
+	child, ok := de.subDirectories[dirName]
+	return child, ok
+}
+func (de *DirectoryEntryInMap) addChild(dirName string, child *DirectoryEntryInMap) {
+	de.Lock()
+	defer de.Unlock()
+	de.subDirectories[dirName] = child
+}
+func (de *DirectoryEntryInMap) removeChild(dirName string) {
+	de.Lock()
+	defer de.Unlock()
+	delete(de.subDirectories, dirName)
+}
+func (de *DirectoryEntryInMap) hasChildren() bool {
+	de.Lock()
+	defer de.Unlock()
+	return len(de.subDirectories) > 0
+}
+func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryEntry) {
+	de.Lock()
+	defer de.Unlock()
+	for k, v := range de.subDirectories {
+		dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id})
+	}
+	return dirNames
+}
+
 type DirectoryManagerInMap struct {
 	Root      *DirectoryEntryInMap
 	max       filer.DirectoryId
@@ -30,10 +61,8 @@ type DirectoryManagerInMap struct {
 	isLoading bool
 }
 
-func (dm *DirectoryManagerInMap) NewDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap, err error) {
-	writeLock.Lock()
-	defer writeLock.Unlock()
-	d = &DirectoryEntryInMap{Name: name, Parent: parent, SubDirectories: make(map[string]*DirectoryEntryInMap)}
+func (dm *DirectoryManagerInMap) newDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap, err error) {
+	d = &DirectoryEntryInMap{Name: name, Parent: parent, subDirectories: make(map[string]*DirectoryEntryInMap)}
 	var parts []string
 	for p := d; p != nil && p.Name != ""; p = p.Parent {
 		parts = append(parts, p.Name)
@@ -59,10 +88,10 @@ func (dm *DirectoryManagerInMap) log(words ...string) {
 
 func NewDirectoryManagerInMap(dirLogFile string) (dm *DirectoryManagerInMap, err error) {
 	dm = &DirectoryManagerInMap{}
-	//dm.Root do not use NewDirectoryEntryInMap, since dm.max will be changed
-	dm.Root = &DirectoryEntryInMap{SubDirectories: make(map[string]*DirectoryEntryInMap)}
+	//dm.Root do not use newDirectoryEntryInMap, since dm.max will be changed
+	dm.Root = &DirectoryEntryInMap{subDirectories: make(map[string]*DirectoryEntryInMap)}
 	if dm.logFile, err = os.OpenFile(dirLogFile, os.O_RDWR|os.O_CREATE, 0644); err != nil {
-		return nil, fmt.Errorf("cannot write directory log file %s.idx: %v", dirLogFile, err)
+		return nil, fmt.Errorf("cannot write directory log file %s: %v", dirLogFile, err)
 	}
 	return dm, dm.load()
 }
@@ -128,14 +157,14 @@ func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryI
 	if dirPath == "" {
 		return dm.Root, nil
 	}
-	dirPath = filepath.Clean(dirPath)
+	dirPath = CleanFilePath(dirPath)
 	if dirPath == "/" {
 		return dm.Root, nil
 	}
 	parts := strings.Split(dirPath, "/")
 	dir := dm.Root
 	for i := 1; i < len(parts); i++ {
-		if sub, ok := dir.SubDirectories[parts[i]]; ok {
+		if sub, ok := dir.getChild(parts[i]); ok {
 			dir = sub
 		} else {
 			return dm.Root, fmt.Errorf("Directory %s Not Found", dirPath)
@@ -152,27 +181,37 @@ func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryI
 }
 
 func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error {
-	dirPath = filepath.Clean(dirPath)
+	dirPath = CleanFilePath(dirPath)
 	if dirPath == "/" {
 		return nil
 	}
 	parts := strings.Split(dirPath, "/")
 	dir := dm.Root
 	for i := 1; i < len(parts); i++ {
-		sub, ok := dir.SubDirectories[parts[i]]
+		sub, ok := dir.getChild(parts[i])
 		if !ok {
-			if i != len(parts)-1 {
-				return fmt.Errorf("%s should be created after parent %s", dirPath, parts[i])
+			writeLock.Lock()
+			if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread {
+				sub = sub2
+			} else {
+				if i != len(parts)-1 {
+					writeLock.Unlock()
+					return fmt.Errorf("%s should be created after parent %s", dirPath, parts[i])
+				}
+				var err error
+				sub, err = dm.newDirectoryEntryInMap(dir, parts[i])
+				if err != nil {
+					writeLock.Unlock()
+					return err
+				}
+				if sub.Id != dirId {
+					writeLock.Unlock()
+					// the dir.log should be the same order as in-memory directory id
+					return fmt.Errorf("%s should be have id %v instead of %v", dirPath, sub.Id, dirId)
+				}
+				dir.addChild(parts[i], sub)
 			}
-			var err error
-			sub, err = dm.NewDirectoryEntryInMap(dir, parts[i])
-			if err != nil {
-				return err
-			}
-			if sub.Id != dirId {
-				return fmt.Errorf("%s should be have id %v instead of %v", dirPath, sub.Id, dirId)
-			}
-			dir.SubDirectories[parts[i]] = sub
+			writeLock.Unlock()
 		}
 		dir = sub
 	}
@@ -180,22 +219,29 @@ func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.Direc
 }
 
 func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEntryInMap, created bool) {
-	dirPath = filepath.Clean(dirPath)
+	dirPath = CleanFilePath(dirPath)
 	if dirPath == "/" {
 		return dm.Root, false
 	}
 	parts := strings.Split(dirPath, "/")
 	dir = dm.Root
 	for i := 1; i < len(parts); i++ {
-		sub, ok := dir.SubDirectories[parts[i]]
+		sub, ok := dir.getChild(parts[i])
 		if !ok {
-			var err error
-			sub, err = dm.NewDirectoryEntryInMap(dir, parts[i])
-			if err != nil {
-				return nil, false
+			writeLock.Lock()
+			if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread {
+				sub = sub2
+			} else {
+				var err error
+				sub, err = dm.newDirectoryEntryInMap(dir, parts[i])
+				if err != nil {
+					writeLock.Unlock()
+					return nil, false
+				}
+				dir.addChild(parts[i], sub)
+				created = true
 			}
-			dir.SubDirectories[parts[i]] = sub
-			created = true
+			writeLock.Unlock()
 		}
 		dir = sub
 	}
@@ -219,11 +265,11 @@ func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParent
 		return pe
 	}
 	dm.log("mov", oldDirPath, newParentDirPath, newName)
-	delete(oldDir.Parent.SubDirectories, oldDir.Name)
+	oldDir.Parent.removeChild(oldDir.Name)
 	if newName == "" {
 		newName = oldDir.Name
 	}
-	parentDir.SubDirectories[newName] = oldDir
+	parentDir.addChild(newName, oldDir)
 	oldDir.Name = newName
 	oldDir.Parent = parentDir
 	return nil
@@ -234,10 +280,7 @@ func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []fil
 	if e != nil {
 		return dirNames, e
 	}
-	for k, v := range d.SubDirectories {
-		dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id})
-	}
-	return dirNames, nil
+	return d.children(), nil
 }
 func (dm *DirectoryManagerInMap) DeleteDirectory(dirPath string) error {
 	writeLock.Lock()
@@ -249,11 +292,19 @@ func (dm *DirectoryManagerInMap) DeleteDirectory(dirPath string) error {
 	if e != nil {
 		return e
 	}
-	if len(d.SubDirectories) != 0 {
+	if d.hasChildren() {
 		return fmt.Errorf("dir %s still has sub directories", dirPath)
 	}
-	delete(d.Parent.SubDirectories, d.Name)
+	d.Parent.removeChild(d.Name)
 	d.Parent = nil
 	dm.log("del", dirPath)
 	return nil
 }
+
+func CleanFilePath(fp string) string {
+	ret := filepath.Clean(fp)
+	if os.PathSeparator == '\\' {
+		return strings.Replace(ret, "\\", "/", -1)
+	}
+	return ret
+}

weed/filer/embedded_filer/directory_test.go

@@ -57,13 +57,13 @@ func TestDirectory(t *testing.T) {
 func printTree(node *DirectoryEntryInMap, path []string) {
 	println(strings.Join(path, "/") + "/" + node.Name)
 	path = append(path, node.Name)
-	for _, v := range node.SubDirectories {
+	for _, v := range node.subDirectories {
 		printTree(v, path)
 	}
 }
 
 func compare(root1 *DirectoryEntryInMap, root2 *DirectoryEntryInMap) bool {
-	if len(root1.SubDirectories) != len(root2.SubDirectories) {
+	if len(root1.subDirectories) != len(root2.subDirectories) {
 		return false
 	}
 	if root1.Name != root2.Name {
@@ -77,8 +77,8 @@ func compare(root1 *DirectoryEntryInMap, root2 *DirectoryEntryInMap) bool {
 			return false
 		}
 	}
-	for k, v := range root1.SubDirectories {
-		if !compare(v, root2.SubDirectories[k]) {
+	for k, v := range root1.subDirectories {
+		if !compare(v, root2.subDirectories[k]) {
 			return false
 		}
 	}

weed/filer/embedded_filer/filer_embedded.go

@@ -6,8 +6,8 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/chrislusf/seaweedfs/go/filer"
-	"github.com/chrislusf/seaweedfs/go/operation"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/operation"
 )
 
 type FilerEmbedded struct {

weed/filer/embedded_filer/files_in_leveldb.go

@@ -3,8 +3,8 @@ package embedded_filer
 import (
 	"bytes"
 
-	"github.com/chrislusf/seaweedfs/go/filer"
-	"github.com/chrislusf/seaweedfs/go/glog"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
@@ -43,6 +43,9 @@ func (fl *FileListInLevelDb) CreateFile(dirId filer.DirectoryId, fileName string
 }
 func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
 	if fid, err = fl.FindFile(dirId, fileName); err != nil {
+		if err == leveldb.ErrNotFound {
+			return "", nil
+		}
 		return
 	}
 	err = fl.db.Delete(genKey(dirId, fileName), nil)
@@ -50,7 +53,9 @@ func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string
 }
 func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
 	data, e := fl.db.Get(genKey(dirId, fileName), nil)
-	if e != nil {
+	if e == leveldb.ErrNotFound {
+		return "", filer.ErrNotFound
+	} else if e != nil {
 		return "", e
 	}
 	return string(data), nil

weed/filer/filer.go

@@ -1,5 +1,9 @@
 package filer
 
+import (
+	"errors"
+)
+
 type FileId string //file id in SeaweedFS
 
 type FileEntry struct {
@@ -26,3 +30,5 @@ type Filer interface {
 	DeleteDirectory(dirPath string, recursive bool) (err error)
 	Move(fromPath string, toPath string) (err error)
 }
+
+var ErrNotFound = errors.New("filer: no entry is found in filer store")

weed/filer/flat_namespace/flat_namespace_filer.go

@@ -3,7 +3,7 @@ package flat_namespace
 import (
 	"errors"
 
-	"github.com/chrislusf/seaweedfs/go/filer"
+	"github.com/chrislusf/seaweedfs/weed/filer"
 )
 
 type FlatNamespaceFiler struct {
@@ -42,7 +42,19 @@ func (filer *FlatNamespaceFiler) DeleteDirectory(dirPath string, recursive bool)
 }
 
 func (filer *FlatNamespaceFiler) DeleteFile(fullFileName string) (fid string, err error) {
-	return filer.store.Delete(fullFileName)
+	fid, err = filer.FindFile(fullFileName)
+	if err != nil {
+		return "", err
+	}
+
+	err = filer.store.Delete(fullFileName)
+	if err != nil {
+		return "", err
+	}
+
+	return fid, nil
+	//return filer.store.Delete(fullFileName)
+	//are you kidding me!!!!
 }
 
 func (filer *FlatNamespaceFiler) Move(fromPath string, toPath string) error {

weed/filer/flat_namespace/flat_namespace_store.go

@@ -5,5 +5,5 @@ import ()
 type FlatNamespaceStore interface {
 	Put(fullFileName string, fid string) (err error)
 	Get(fullFileName string) (fid string, err error)
-	Delete(fullFileName string) (fid string, err error)
+	Delete(fullFileName string) (err error)
 }

weed/filer/mysql_store/README.md

@@ -0,0 +1,67 @@
+#MySQL filer mapping store
+
+## Schema format
+
+
+Basically, uriPath and fid are the key elements stored in MySQL. In view of the optimization and user's usage, 
+adding primary key with integer type and involving createTime, updateTime, status fields should be somewhat meaningful.
+Of course, you could customize the schema per your concretely circumstance freely.
+
+<pre><code>
+CREATE TABLE IF NOT EXISTS `filer_mapping` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `uriPath` char(256) NOT NULL DEFAULT "" COMMENT 'http uriPath',
+  `fid` char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid',
+  `createTime` int(10) NOT NULL DEFAULT 0 COMMENT 'createdTime in unix timestamp',
+  `updateTime` int(10) NOT NULL DEFAULT 0 COMMENT 'updatedTime in unix timestamp',
+  `remark` varchar(20) NOT NULL DEFAULT "" COMMENT 'reserverd field',
+  `status` tinyint(2) DEFAULT '1' COMMENT 'resource status',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `index_uriPath` (`uriPath`)
+) DEFAULT CHARSET=utf8;
+</code></pre>
+
+
+The MySQL 's config params is not added into the weed command option as other stores(redis,cassandra). Instead,
+We created a config file(json format) for them. TOML,YAML or XML also should be OK. But TOML and YAML need import thirdparty package
+while XML is a little bit complex.
+
+The sample config file's  content is below:
+
+<pre><code>
+{
+    "mysql": [
+        {
+            "User": "root",
+            "Password": "root",
+            "HostName": "127.0.0.1",
+            "Port": 3306,
+            "DataBase": "seaweedfs"
+        },
+        {
+            "User": "root",
+            "Password": "root",
+            "HostName": "127.0.0.2",
+            "Port": 3306,
+            "DataBase": "seaweedfs"
+        }
+    ],
+    "IsSharding":true,
+    "ShardCount":1024
+}
+</code></pre>
+
+
+The "mysql" field in above conf file is an array which include all mysql instances you prepared to store sharding data.
+
+1. If one mysql instance is enough, just keep one instance in "mysql" field.
+
+2. If table sharding at a specific mysql instance is needed , mark "IsSharding" field with true and specify total table sharding numbers using "ShardCount" field.
+
+3. If the mysql service could be auto scaled transparently in your environment, just config one mysql instance(usually it's a frondend proxy or VIP),and mark "IsSharding" with false value
+
+4. If you prepare more than one mysql instance and have no plan to use table sharding for any instance(mark isSharding with false), instance sharding will still be done implicitly
+
+
+
+

weed/filer/mysql_store/mysql_store.go

@@ -0,0 +1,270 @@
+package mysql_store
+
+import (
+	"database/sql"
+	"fmt"
+	"hash/crc32"
+	"sync"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+const (
+	sqlUrl                     = "%s:%s@tcp(%s:%d)/%s?charset=utf8"
+	default_maxIdleConnections = 100
+	default_maxOpenConnections = 50
+	default_maxTableNums       = 1024
+	tableName                  = "filer_mapping"
+)
+
+var (
+	_init_db        sync.Once
+	_db_connections []*sql.DB
+)
+
+type MySqlConf struct {
+	User               string
+	Password           string
+	HostName           string
+	Port               int
+	DataBase           string
+	MaxIdleConnections int
+	MaxOpenConnections int
+}
+
+type ShardingConf struct {
+	IsSharding bool `json:"isSharding"`
+	ShardCount int  `json:"shardCount"`
+}
+
+type MySqlStore struct {
+	dbs        []*sql.DB
+	isSharding bool
+	shardCount int
+}
+
+func getDbConnection(confs []MySqlConf) []*sql.DB {
+	_init_db.Do(func() {
+		for _, conf := range confs {
+
+			sqlUrl := fmt.Sprintf(sqlUrl, conf.User, conf.Password, conf.HostName, conf.Port, conf.DataBase)
+			var dbErr error
+			_db_connection, dbErr := sql.Open("mysql", sqlUrl)
+			if dbErr != nil {
+				_db_connection.Close()
+				_db_connection = nil
+				panic(dbErr)
+			}
+			var maxIdleConnections, maxOpenConnections int
+
+			if conf.MaxIdleConnections != 0 {
+				maxIdleConnections = conf.MaxIdleConnections
+			} else {
+				maxIdleConnections = default_maxIdleConnections
+			}
+			if conf.MaxOpenConnections != 0 {
+				maxOpenConnections = conf.MaxOpenConnections
+			} else {
+				maxOpenConnections = default_maxOpenConnections
+			}
+
+			_db_connection.SetMaxIdleConns(maxIdleConnections)
+			_db_connection.SetMaxOpenConns(maxOpenConnections)
+			_db_connections = append(_db_connections, _db_connection)
+		}
+	})
+	return _db_connections
+}
+
+func NewMysqlStore(confs []MySqlConf, isSharding bool, shardCount int) *MySqlStore {
+	ms := &MySqlStore{
+		dbs:        getDbConnection(confs),
+		isSharding: isSharding,
+		shardCount: shardCount,
+	}
+
+	for _, db := range ms.dbs {
+		if !isSharding {
+			ms.shardCount = 1
+		} else {
+			if ms.shardCount == 0 {
+				ms.shardCount = default_maxTableNums
+			}
+		}
+		for i := 0; i < ms.shardCount; i++ {
+			if err := ms.createTables(db, tableName, i); err != nil {
+				fmt.Printf("create table failed %v", err)
+			}
+		}
+	}
+
+	return ms
+}
+
+func (s *MySqlStore) hash(fullFileName string) (instance_offset, table_postfix int) {
+	hash_value := crc32.ChecksumIEEE([]byte(fullFileName))
+	instance_offset = int(hash_value) % len(s.dbs)
+	table_postfix = int(hash_value) % s.shardCount
+	return
+}
+
+func (s *MySqlStore) parseFilerMappingInfo(path string) (instanceId int, tableFullName string, err error) {
+	instance_offset, table_postfix := s.hash(path)
+	instanceId = instance_offset
+	if s.isSharding {
+		tableFullName = fmt.Sprintf("%s_%04d", tableName, table_postfix)
+	} else {
+		tableFullName = tableName
+	}
+	return
+}
+
+func (s *MySqlStore) Get(fullFilePath string) (fid string, err error) {
+	instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath)
+	if err != nil {
+		return "", fmt.Errorf("MySqlStore Get operation can not parse file path %s: err is %v", fullFilePath, err)
+	}
+	fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName)
+	if err == sql.ErrNoRows {
+		//Could not found
+		err = filer.ErrNotFound
+	}
+	return fid, err
+}
+
+func (s *MySqlStore) Put(fullFilePath string, fid string) (err error) {
+	var tableFullName string
+
+	instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath)
+	if err != nil {
+		return fmt.Errorf("MySqlStore Put operation can not parse file path %s: err is %v", fullFilePath, err)
+	}
+	var old_fid string
+	if old_fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil && err != sql.ErrNoRows {
+		return fmt.Errorf("MySqlStore Put operation failed when querying path %s: err is %v", fullFilePath, err)
+	} else {
+		if len(old_fid) == 0 {
+			err = s.insert(fullFilePath, fid, s.dbs[instance_offset], tableFullName)
+			err = fmt.Errorf("MySqlStore Put operation failed when inserting path %s with fid %s : err is %v", fullFilePath, fid, err)
+		} else {
+			err = s.update(fullFilePath, fid, s.dbs[instance_offset], tableFullName)
+			err = fmt.Errorf("MySqlStore Put operation failed when updating path %s with fid %s : err is %v", fullFilePath, fid, err)
+		}
+	}
+	return
+}
+
+func (s *MySqlStore) Delete(fullFilePath string) (err error) {
+	var fid string
+	instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath)
+	if err != nil {
+		return fmt.Errorf("MySqlStore Delete operation can not parse file path %s: err is %v", fullFilePath, err)
+	}
+	if fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil {
+		return fmt.Errorf("MySqlStore Delete operation failed when querying path %s: err is %v", fullFilePath, err)
+	} else if fid == "" {
+		return nil
+	}
+	if err = s.delete(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil {
+		return fmt.Errorf("MySqlStore Delete operation failed when deleting path %s: err is %v", fullFilePath, err)
+	} else {
+		return nil
+	}
+}
+
+func (s *MySqlStore) Close() {
+	for _, db := range s.dbs {
+		db.Close()
+	}
+}
+
+var createTable = `
+CREATE TABLE IF NOT EXISTS %s (
+  id bigint(20) NOT NULL AUTO_INCREMENT,
+  uriPath char(256) NOT NULL DEFAULT "" COMMENT 'http uriPath',
+  fid char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid',
+  createTime int(10) NOT NULL DEFAULT 0 COMMENT 'createdTime in unix timestamp',
+  updateTime int(10) NOT NULL DEFAULT 0 COMMENT 'updatedTime in unix timestamp',
+  remark varchar(20) NOT NULL DEFAULT "" COMMENT 'reserverd field',
+  status tinyint(2) DEFAULT '1' COMMENT 'resource status',
+  PRIMARY KEY (id),
+  UNIQUE KEY index_uriPath (uriPath)
+) DEFAULT CHARSET=utf8;
+`
+
+func (s *MySqlStore) createTables(db *sql.DB, tableName string, postfix int) error {
+	var realTableName string
+	if s.isSharding {
+		realTableName = fmt.Sprintf("%s_%4d", tableName, postfix)
+	} else {
+		realTableName = tableName
+	}
+
+	stmt, err := db.Prepare(fmt.Sprintf(createTable, realTableName))
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+
+	_, err = stmt.Exec()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *MySqlStore) query(uriPath string, db *sql.DB, tableName string) (string, error) {
+	sqlStatement := "SELECT fid FROM %s WHERE uriPath=?"
+	row := db.QueryRow(fmt.Sprintf(sqlStatement, tableName), uriPath)
+	var fid string
+	err := row.Scan(&fid)
+	if err != nil {
+		return "", err
+	}
+	return fid, nil
+}
+
+func (s *MySqlStore) update(uriPath string, fid string, db *sql.DB, tableName string) error {
+	sqlStatement := "UPDATE %s SET fid=?, updateTime=? WHERE uriPath=?"
+	res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), fid, time.Now().Unix(), uriPath)
+	if err != nil {
+		return err
+	}
+
+	_, err = res.RowsAffected()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *MySqlStore) insert(uriPath string, fid string, db *sql.DB, tableName string) error {
+	sqlStatement := "INSERT INTO %s (uriPath,fid,createTime) VALUES(?,?,?)"
+	res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), uriPath, fid, time.Now().Unix())
+	if err != nil {
+		return err
+	}
+
+	_, err = res.RowsAffected()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *MySqlStore) delete(uriPath string, db *sql.DB, tableName string) error {
+	sqlStatement := "DELETE FROM %s WHERE uriPath=?"
+	res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), uriPath)
+	if err != nil {
+		return err
+	}
+
+	_, err = res.RowsAffected()
+	if err != nil {
+		return err
+	}
+	return nil
+}

weed/filer/mysql_store/mysql_store_test.go

@@ -0,0 +1,30 @@
+package mysql_store
+
+import (
+	"encoding/json"
+	"hash/crc32"
+	"testing"
+)
+
+func TestGenerateMysqlConf(t *testing.T) {
+	var conf []MySqlConf
+	conf = append(conf, MySqlConf{
+		User:     "root",
+		Password: "root",
+		HostName: "localhost",
+		Port:     3306,
+		DataBase: "seaweedfs",
+	})
+	body, err := json.Marshal(conf)
+	if err != nil {
+		t.Errorf("json encoding err %s", err.Error())
+	}
+	t.Logf("json output is %s", string(body))
+}
+
+func TestCRC32FullPathName(t *testing.T) {
+	fullPathName := "/prod-bucket/law632191483895612493300-signed.pdf"
+	hash_value := crc32.ChecksumIEEE([]byte(fullPathName))
+	table_postfix := int(hash_value) % 1024
+	t.Logf("table postfix %d", table_postfix)
+}

weed/filer/postgres_store/postgres_store.go

@@ -0,0 +1,623 @@
+package postgres_store
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+
+	_ "github.com/lib/pq"
+	_ "path/filepath"
+	"strings"
+)
+
+const (
+	default_maxIdleConnections = 100
+	default_maxOpenConnections = 50
+	filesTableName             = "files"
+	directoriesTableName       = "directories"
+)
+
+var (
+	_init_db       sync.Once
+	_db_connection *sql.DB
+)
+
+type PostgresConf struct {
+	User               string
+	Password           string
+	HostName           string
+	Port               int
+	DataBase           string
+	SslMode            string
+	MaxIdleConnections int
+	MaxOpenConnections int
+}
+
+type PostgresStore struct {
+	db       *sql.DB
+	server   string
+	user     string
+	password string
+}
+
+func (s *PostgresStore) CreateFile(fullFileName string, fid string) (err error) {
+	glog.V(3).Infoln("Calling posgres_store CreateFile")
+	return s.Put(fullFileName, fid)
+}
+
+func (s *PostgresStore) FindFile(fullFileName string) (fid string, err error) {
+	glog.V(3).Infoln("Calling posgres_store FindFile")
+	return s.Get(fullFileName)
+}
+
+func (s *PostgresStore) DeleteFile(fullFileName string) (fid string, err error) {
+	glog.V(3).Infoln("Calling posgres_store DeleteFile")
+	return "", s.Delete(fullFileName)
+}
+
+func (s *PostgresStore) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
+	glog.V(3).Infoln("Calling posgres_store FindDirectory")
+	return s.FindDir(dirPath)
+}
+
+func (s *PostgresStore) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
+	glog.V(3).Infoln("Calling posgres_store ListDirectories")
+	return s.ListDirs(dirPath)
+}
+
+func (s *PostgresStore) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
+	glog.V(3).Infoln("Calling posgres_store ListFiles")
+	return s.FindFiles(dirPath, lastFileName, limit)
+}
+
+func (s *PostgresStore) DeleteDirectory(dirPath string, recursive bool) (err error) {
+	glog.V(3).Infoln("Calling posgres_store DeleteDirectory")
+	return s.DeleteDir(dirPath, recursive)
+}
+
+func (s *PostgresStore) Move(fromPath string, toPath string) (err error) {
+	glog.V(3).Infoln("Calling posgres_store Move")
+	return errors.New("Move is not yet implemented for the PostgreSQL store.")
+}
+
+func databaseExists(db *sql.DB, databaseName string) (bool, error) {
+	sqlStatement := "SELECT datname from pg_database WHERE datname='%s'"
+	row := db.QueryRow(fmt.Sprintf(sqlStatement, databaseName))
+
+	var dbName string
+	err := row.Scan(&dbName)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+func createDatabase(db *sql.DB, databaseName string) error {
+	sqlStatement := "CREATE DATABASE %s ENCODING='UTF8'"
+	_, err := db.Exec(fmt.Sprintf(sqlStatement, databaseName))
+	return err
+}
+
+func getDbConnection(conf PostgresConf) *sql.DB {
+	_init_db.Do(func() {
+
+		sqlUrl := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30", conf.HostName, conf.Port, conf.User, conf.Password, "postgres", conf.SslMode)
+		glog.V(3).Infoln("Opening postgres master database")
+
+		var dbErr error
+		_db_connection, dbErr := sql.Open("postgres", sqlUrl)
+		if dbErr != nil {
+			_db_connection.Close()
+			_db_connection = nil
+			panic(dbErr)
+		}
+
+		pingErr := _db_connection.Ping()
+		if pingErr != nil {
+			_db_connection.Close()
+			_db_connection = nil
+			panic(pingErr)
+		}
+
+		glog.V(3).Infoln("Checking to see if DB exists: ", conf.DataBase)
+		var existsErr error
+		dbExists, existsErr := databaseExists(_db_connection, conf.DataBase)
+		if existsErr != nil {
+			_db_connection.Close()
+			_db_connection = nil
+			panic(existsErr)
+		}
+
+		if !dbExists {
+			glog.V(3).Infoln("Database doesn't exist. Attempting to create one:  ", conf.DataBase)
+			createErr := createDatabase(_db_connection, conf.DataBase)
+			if createErr != nil {
+				_db_connection.Close()
+				_db_connection = nil
+				panic(createErr)
+			}
+		}
+
+		glog.V(3).Infoln("Closing master postgres database and opening configured database: ", conf.DataBase)
+		_db_connection.Close()
+		_db_connection = nil
+
+		sqlUrl = fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30", conf.HostName, conf.Port, conf.User, conf.Password, conf.DataBase, conf.SslMode)
+		_db_connection, dbErr = sql.Open("postgres", sqlUrl)
+		if dbErr != nil {
+			_db_connection.Close()
+			_db_connection = nil
+			panic(dbErr)
+		}
+
+		pingErr = _db_connection.Ping()
+		if pingErr != nil {
+			_db_connection.Close()
+			_db_connection = nil
+			panic(pingErr)
+		}
+
+		var maxIdleConnections, maxOpenConnections int
+
+		if conf.MaxIdleConnections != 0 {
+			maxIdleConnections = conf.MaxIdleConnections
+		} else {
+			maxIdleConnections = default_maxIdleConnections
+		}
+		if conf.MaxOpenConnections != 0 {
+			maxOpenConnections = conf.MaxOpenConnections
+		} else {
+			maxOpenConnections = default_maxOpenConnections
+		}
+
+		_db_connection.SetMaxIdleConns(maxIdleConnections)
+		_db_connection.SetMaxOpenConns(maxOpenConnections)
+	})
+	return _db_connection
+}
+
+//func NewPostgresStore(master string, confs []PostgresConf, isSharding bool, shardCount int) *PostgresStore {
+func NewPostgresStore(master string, conf PostgresConf) *PostgresStore {
+	pg := &PostgresStore{
+		db: getDbConnection(conf),
+	}
+
+	pg.createDirectoriesTable()
+
+	if err := pg.createFilesTable(); err != nil {
+		fmt.Printf("create table failed %v", err)
+	}
+
+	return pg
+}
+
+func (s *PostgresStore) Get(fullFilePath string) (fid string, err error) {
+	if err != nil {
+		return "", fmt.Errorf("PostgresStore Get operation can not parse file path %s: err is %v", fullFilePath, err)
+	}
+	fid, err = s.query(fullFilePath)
+
+	return fid, err
+}
+
+func (s *PostgresStore) Put(fullFilePath string, fid string) (err error) {
+	var old_fid string
+	if old_fid, err = s.query(fullFilePath); err != nil && err != sql.ErrNoRows {
+		return fmt.Errorf("PostgresStore Put operation failed when querying path %s: err is %v", fullFilePath, err)
+	} else {
+		if len(old_fid) == 0 {
+			err = s.insert(fullFilePath, fid)
+			if err != nil {
+				return fmt.Errorf("PostgresStore Put operation failed when inserting path %s with fid %s : err is %v", fullFilePath, fid, err)
+			}
+		} else {
+			err = s.update(fullFilePath, fid)
+			if err != nil {
+				return fmt.Errorf("PostgresStore Put operation failed when updating path %s with fid %s : err is %v", fullFilePath, fid, err)
+			}
+		}
+	}
+	return
+}
+
+func (s *PostgresStore) Delete(fullFilePath string) (err error) {
+	var fid string
+	if err != nil {
+		return fmt.Errorf("PostgresStore Delete operation can not parse file path %s: err is %v", fullFilePath, err)
+	}
+	if fid, err = s.query(fullFilePath); err != nil {
+		return fmt.Errorf("PostgresStore Delete operation failed when querying path %s: err is %v", fullFilePath, err)
+	} else if fid == "" {
+		return nil
+	}
+	if err = s.delete(fullFilePath); err != nil {
+		return fmt.Errorf("PostgresStore Delete operation failed when deleting path %s: err is %v", fullFilePath, err)
+	} else {
+		return nil
+	}
+}
+
+func (s *PostgresStore) Close() {
+	s.db.Close()
+}
+
+func (s *PostgresStore) FindDir(dirPath string) (dirId filer.DirectoryId, err error) {
+	dirId, _, err = s.lookupDirectory(dirPath)
+	return dirId, err
+}
+
+func (s *PostgresStore) ListDirs(dirPath string) (dirs []filer.DirectoryEntry, err error) {
+	dirs, err = s.findDirectories(dirPath, 20)
+
+	glog.V(3).Infof("Postgres ListDirs = found %d directories under %s", len(dirs), dirPath)
+
+	return dirs, err
+}
+
+func (s *PostgresStore) DeleteDir(dirPath string, recursive bool) (err error) {
+	err = s.deleteDirectory(dirPath, recursive)
+	if err != nil {
+		glog.V(0).Infof("Error in Postgres DeleteDir '%s' (recursive = '%t'): %s", err)
+	}
+	return err
+}
+
+func (s *PostgresStore) FindFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
+	files, err = s.findFiles(dirPath, lastFileName, limit)
+
+	return files, err
+}
+
+var createDirectoryTable = `
+
+CREATE TABLE IF NOT EXISTS %s (
+  id BIGSERIAL NOT NULL,
+  directoryRoot VARCHAR(1024) NOT NULL DEFAULT '',
+  directoryName VARCHAR(1024) NOT NULL DEFAULT '',
+  CONSTRAINT unique_directory UNIQUE (directoryRoot, directoryName)
+);
+`
+
+var createFileTable = `
+
+CREATE TABLE IF NOT EXISTS %s (
+  id BIGSERIAL NOT NULL,
+  directoryPart VARCHAR(1024) NOT NULL DEFAULT '',
+  filePart VARCHAR(1024) NOT NULL DEFAULT '',
+  fid VARCHAR(36) NOT NULL DEFAULT '',
+  createTime BIGINT NOT NULL DEFAULT 0,
+  updateTime BIGINT NOT NULL DEFAULT 0,
+  remark VARCHAR(20) NOT NULL DEFAULT '',
+  status SMALLINT NOT NULL DEFAULT '1',
+  PRIMARY KEY (id),
+  CONSTRAINT %s_unique_file UNIQUE (directoryPart, filePart)
+);
+`
+
+func (s *PostgresStore) createDirectoriesTable() error {
+	glog.V(3).Infoln("Creating postgres table if it doesn't exist: ", directoriesTableName)
+
+	sqlCreate := fmt.Sprintf(createDirectoryTable, directoriesTableName)
+
+	stmt, err := s.db.Prepare(sqlCreate)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+
+	_, err = stmt.Exec()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *PostgresStore) createFilesTable() error {
+
+	glog.V(3).Infoln("Creating postgres table if it doesn't exist: ", filesTableName)
+
+	sqlCreate := fmt.Sprintf(createFileTable, filesTableName, filesTableName)
+
+	stmt, err := s.db.Prepare(sqlCreate)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+
+	_, err = stmt.Exec()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *PostgresStore) query(uriPath string) (string, error) {
+	directoryPart, filePart := filepath.Split(uriPath)
+	sqlStatement := fmt.Sprintf("SELECT fid FROM %s WHERE directoryPart=$1 AND filePart=$2", filesTableName)
+
+	row := s.db.QueryRow(sqlStatement, directoryPart, filePart)
+	var fid string
+	err := row.Scan(&fid)
+
+	glog.V(3).Infof("Postgres query -- looking up path '%s' and found id '%s' ", uriPath, fid)
+
+	if err != nil {
+		return "", err
+	}
+	return fid, nil
+}
+
+func (s *PostgresStore) update(uriPath string, fid string) error {
+	directoryPart, filePart := filepath.Split(uriPath)
+	sqlStatement := fmt.Sprintf("UPDATE %s SET fid=$1, updateTime=$2 WHERE directoryPart=$3 AND filePart=$4", filesTableName)
+
+	glog.V(3).Infof("Postgres query -- updating path '%s' with id '%s'", uriPath, fid)
+
+	res, err := s.db.Exec(sqlStatement, fid, time.Now().Unix(), directoryPart, filePart)
+	if err != nil {
+		return err
+	}
+
+	_, err = res.RowsAffected()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *PostgresStore) insert(uriPath string, fid string) error {
+	directoryPart, filePart := filepath.Split(uriPath)
+
+	existingId, _, _ := s.lookupDirectory(directoryPart)
+	if existingId == 0 {
+		s.recursiveInsertDirectory(directoryPart)
+	}
+
+	sqlStatement := fmt.Sprintf("INSERT INTO %s (directoryPart,filePart,fid,createTime) VALUES($1, $2, $3, $4)", filesTableName)
+	glog.V(3).Infof("Postgres query -- inserting path '%s' with id '%s'", uriPath, fid)
+
+	res, err := s.db.Exec(sqlStatement, directoryPart, filePart, fid, time.Now().Unix())
+
+	if err != nil {
+		return err
+	}
+
+	rows, err := res.RowsAffected()
+	if rows != 1 {
+		return fmt.Errorf("Postgres insert -- rows affected = %d. Expecting 1", rows)
+	}
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *PostgresStore) recursiveInsertDirectory(dirPath string) {
+	pathParts := strings.Split(dirPath, "/")
+
+	var workingPath string = "/"
+	for _, part := range pathParts {
+		if part == "" {
+			continue
+		}
+		workingPath += (part + "/")
+		existingId, _, _ := s.lookupDirectory(workingPath)
+		if existingId == 0 {
+			s.insertDirectory(workingPath)
+		}
+	}
+}
+
+func (s *PostgresStore) insertDirectory(dirPath string) {
+	pathParts := strings.Split(dirPath, "/")
+
+	directoryRoot := "/"
+	directoryName := ""
+	if len(pathParts) > 1 {
+		directoryRoot = strings.Join(pathParts[0:len(pathParts)-2], "/") + "/"
+		directoryName = strings.Join(pathParts[len(pathParts)-2:], "/")
+	} else if len(pathParts) == 1 {
+		directoryRoot = "/"
+		directoryName = pathParts[0] + "/"
+	}
+	sqlInsertDirectoryStatement := fmt.Sprintf("INSERT INTO %s (directoryroot, directoryname)  "+
+		"SELECT $1, $2 WHERE NOT EXISTS ( SELECT id FROM %s WHERE directoryroot=$3 AND directoryname=$4 )",
+		directoriesTableName, directoriesTableName)
+
+	glog.V(4).Infof("Postgres query -- Inserting directory (if it doesn't exist) - root = %s, name = %s",
+		directoryRoot, directoryName)
+
+	_, err := s.db.Exec(sqlInsertDirectoryStatement, directoryRoot, directoryName, directoryRoot, directoryName)
+	if err != nil {
+		glog.V(0).Infof("Postgres query -- Error inserting directory - root = %s, name = %s: %s",
+			directoryRoot, directoryName, err)
+	}
+}
+
+func (s *PostgresStore) delete(uriPath string) error {
+	directoryPart, filePart := filepath.Split(uriPath)
+	sqlStatement := fmt.Sprintf("DELETE FROM %s WHERE directoryPart=$1 AND filePart=$2", filesTableName)
+
+	glog.V(3).Infof("Postgres query -- deleting path '%s'", uriPath)
+
+	res, err := s.db.Exec(sqlStatement, directoryPart, filePart)
+	if err != nil {
+		return err
+	}
+
+	_, err = res.RowsAffected()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *PostgresStore) lookupDirectory(dirPath string) (filer.DirectoryId, string, error) {
+	directoryRoot, directoryName := s.mySplitPath(dirPath)
+
+	sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName)
+
+	row := s.db.QueryRow(sqlStatement, directoryRoot, directoryName)
+	var id filer.DirectoryId
+	var dirRoot string
+	var dirName string
+	err := row.Scan(&id, &dirRoot, &dirName)
+
+	glog.V(3).Infof("Postgres lookupDirectory -- looking up directory '%s' and found id '%d', root '%s', name '%s' ", dirPath, id, dirRoot, dirName)
+
+	if err != nil {
+		return 0, "", err
+	}
+	return id, filepath.Join(dirRoot, dirName), err
+}
+
+func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer.DirectoryEntry, err error) {
+	sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName != '' ORDER BY id LIMIT $2", directoriesTableName)
+	rows, err := s.db.Query(sqlStatement, dirPath, limit)
+
+	if err != nil {
+		glog.V(0).Infof("Postgres findDirectories error: %s", err)
+	}
+
+	if rows != nil {
+		defer rows.Close()
+		for rows.Next() {
+			var id filer.DirectoryId
+			var directoryRoot string
+			var directoryName string
+
+			scanErr := rows.Scan(&id, &directoryRoot, &directoryName)
+			if scanErr != nil {
+				err = scanErr
+			}
+			dirs = append(dirs, filer.DirectoryEntry{Name: (directoryName), Id: id})
+		}
+	}
+	return
+}
+
+func (s *PostgresStore) safeToDeleteDirectory(dirPath string, recursive bool) bool {
+	if recursive {
+		return true
+	}
+	sqlStatement := fmt.Sprintf("SELECT id FROM %s WHERE directoryRoot LIKE $1 LIMIT 1", directoriesTableName)
+	row := s.db.QueryRow(sqlStatement, dirPath+"%")
+
+	var id filer.DirectoryId
+	err := row.Scan(&id)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *PostgresStore) mySplitPath(dirPath string) (directoryRoot string, directoryName string) {
+	pathParts := strings.Split(dirPath, "/")
+	directoryRoot = "/"
+	directoryName = ""
+	if len(pathParts) > 1 {
+		directoryRoot = strings.Join(pathParts[0:len(pathParts)-2], "/") + "/"
+		directoryName = strings.Join(pathParts[len(pathParts)-2:], "/")
+	} else if len(pathParts) == 1 {
+		directoryRoot = "/"
+		directoryName = pathParts[0] + "/"
+	}
+	return directoryRoot, directoryName
+}
+
+func (s *PostgresStore) deleteDirectory(dirPath string, recursive bool) (err error) {
+	directoryRoot, directoryName := s.mySplitPath(dirPath)
+
+	// delete files
+	sqlStatement := fmt.Sprintf("DELETE FROM %s WHERE directorypart=$1", filesTableName)
+	_, err = s.db.Exec(sqlStatement, dirPath)
+	if err != nil {
+		return err
+	}
+
+	// delete specific directory if it is empty or recursive delete was requested
+	safeToDelete := s.safeToDeleteDirectory(dirPath, recursive)
+	if safeToDelete {
+		sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName)
+		_, err = s.db.Exec(sqlStatement, directoryRoot, directoryName)
+		if err != nil {
+			return err
+		}
+	}
+
+	if recursive {
+		// delete descendant files
+		sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directorypart LIKE $1", filesTableName)
+		_, err = s.db.Exec(sqlStatement, dirPath+"%")
+		if err != nil {
+			return err
+		}
+
+		// delete descendant directories
+		sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directoryRoot LIKE $1", directoriesTableName)
+		_, err = s.db.Exec(sqlStatement, dirPath+"%")
+		if err != nil {
+			return err
+		}
+	}
+
+	return err
+}
+
+func (s *PostgresStore) findFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
+	var rows *sql.Rows = nil
+
+	if lastFileName == "" {
+		sqlStatement :=
+			fmt.Sprintf("SELECT fid, directorypart, filepart FROM %s WHERE directorypart=$1 ORDER BY id LIMIT $2", filesTableName)
+		rows, err = s.db.Query(sqlStatement, dirPath, limit)
+	} else {
+		sqlStatement :=
+			fmt.Sprintf("SELECT fid, directorypart, filepart FROM %s WHERE directorypart=$1 "+
+				"AND id > (SELECT id FROM %s WHERE directoryPart=$2 AND filepart=$3)  ORDER BY id LIMIT $4",
+				filesTableName, filesTableName)
+		_, lastFileNameName := filepath.Split(lastFileName)
+		rows, err = s.db.Query(sqlStatement, dirPath, dirPath, lastFileNameName, limit)
+	}
+
+	if err != nil {
+		glog.V(0).Infof("Postgres find files error: %s", err)
+	}
+
+	if rows != nil {
+		defer rows.Close()
+
+		for rows.Next() {
+			var fid filer.FileId
+			var directoryPart string
+			var filePart string
+
+			scanErr := rows.Scan(&fid, &directoryPart, &filePart)
+			if scanErr != nil {
+				err = scanErr
+			}
+
+			files = append(files, filer.FileEntry{Name: filepath.Join(directoryPart, filePart), Id: fid})
+			if len(files) >= limit {
+				break
+			}
+		}
+	}
+
+	glog.V(3).Infof("Postgres findFiles -- looking up files under '%s' and found %d files. Limit=%d, lastFileName=%s",
+		dirPath, len(files), limit, lastFileName)
+
+	return files, err
+}

weed/filer/redis_store/redis_store.go

@@ -1,6 +1,8 @@
 package redis_store
 
 import (
+	"github.com/chrislusf/seaweedfs/weed/filer"
+
 	redis "gopkg.in/redis.v2"
 )
 
@@ -8,10 +10,10 @@ type RedisStore struct {
 	Client *redis.Client
 }
 
-func NewRedisStore(hostPort string, database int) *RedisStore {
+func NewRedisStore(hostPort string, password string, database int) *RedisStore {
 	client := redis.NewTCPClient(&redis.Options{
 		Addr:     hostPort,
-		Password: "", // no password set
+		Password: password,
 		DB:       int64(database),
 	})
 	return &RedisStore{Client: client}
@@ -20,7 +22,7 @@ func NewRedisStore(hostPort string, database int) *RedisStore {
 func (s *RedisStore) Get(fullFileName string) (fid string, err error) {
 	fid, err = s.Client.Get(fullFileName).Result()
 	if err == redis.Nil {
-		err = nil
+		err = filer.ErrNotFound
 	}
 	return fid, err
 }
@@ -33,12 +35,12 @@ func (s *RedisStore) Put(fullFileName string, fid string) (err error) {
 }
 
 // Currently the fid is not returned
-func (s *RedisStore) Delete(fullFileName string) (fid string, err error) {
+func (s *RedisStore) Delete(fullFileName string) (err error) {
 	_, err = s.Client.Del(fullFileName).Result()
 	if err == redis.Nil {
 		err = nil
 	}
-	return "", err
+	return err
 }
 
 func (s *RedisStore) Close() {

weed/glide.lock

@@ -0,0 +1,93 @@
+hash: fc2f4ccfa5d703b62b9c93047bae2295e6926ce1603c2164510c9010ff829945
+updated: 2016-06-24T14:19:43.337337605-07:00
+imports:
+- name: bazil.org/fuse
+  version: 9e8e65293a037ce998ba0df102710c49313aa513
+  subpackages:
+  - fs
+  - fuseutil
+- name: github.com/boltdb/bolt
+  version: 3f7947a25d970e1e5f512276c14d5dcf731ccd5e
+- name: github.com/chrislusf/raft
+  version: 5f7ddd8f479583daf05879d3d3b174aa202c8fb7
+  subpackages:
+  - protobuf
+- name: github.com/dgrijalva/jwt-go
+  version: f0777076321ab64f6efc15a82d9d23b98539b943
+- name: github.com/disintegration/imaging
+  version: d8bbae1de109b518dabc98c6c1633eb358c148a4
+- name: github.com/gocql/gocql
+  version: b7b8a0e04b0cb0ca0b379421c58ec6fab9939b85
+  subpackages:
+  - internal/lru
+  - internal/murmur
+  - internal/streams
+- name: github.com/gogo/protobuf
+  version: 2752d97bbd91927dd1c43296dbf8700e50e2708c
+  subpackages:
+  - proto
+- name: github.com/golang/protobuf
+  version: 0c1f6d65b5a189c2250d10e71a5506f06f9fa0a0
+  subpackages:
+  - proto
+- name: github.com/golang/snappy
+  version: d9eb7a3d35ec988b8585d4a0068e462c27d28380
+- name: github.com/gorilla/context
+  version: aed02d124ae4a0e94fea4541c8effd05bf0c8296
+- name: github.com/gorilla/mux
+  version: 9fa818a44c2bf1396a17f9d5a3c0f6dd39d2ff8e
+- name: github.com/hailocab/go-hostpool
+  version: e80d13ce29ede4452c43dea11e79b9bc8a15b478
+- name: github.com/hashicorp/golang-lru
+  version: a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4
+  subpackages:
+  - simplelru
+- name: github.com/klauspost/crc32
+  version: 19b0b332c9e4516a6370a0456e6182c3b5036720
+- name: github.com/reducedb/encoding
+  version: e31efcfdf1b19f4698709a6f522feaf101b1b1f4
+  subpackages:
+  - cursor
+  - delta/bp32
+  - bitpacking
+- name: github.com/rwcarlsen/goexif
+  version: 709fab3d192d7c62f86043caff1e7e3fb0f42bd8
+  subpackages:
+  - exif
+  - tiff
+- name: github.com/syndtr/goleveldb
+  version: fa5b5c78794bc5c18f330361059f871ae8c2b9d6
+  subpackages:
+  - leveldb
+  - leveldb/util
+  - leveldb/cache
+  - leveldb/comparer
+  - leveldb/errors
+  - leveldb/filter
+  - leveldb/iterator
+  - leveldb/journal
+  - leveldb/memdb
+  - leveldb/opt
+  - leveldb/storage
+  - leveldb/table
+- name: golang.org/x/image
+  version: 97680175a5267bb8b31f1923e7a66df98013b11a
+  subpackages:
+  - bmp
+  - tiff
+  - tiff/lzw
+- name: golang.org/x/net
+  version: bc3663df0ac92f928d419e31e0d2af22e683a5a2
+  subpackages:
+  - context
+- name: golang.org/x/sys
+  version: 62bee037599929a6e9146f29d10dd5208c43507d
+  subpackages:
+  - unix
+- name: gopkg.in/bufio.v1
+  version: 567b2bfa514e796916c4747494d6ff5132a1dfce
+- name: gopkg.in/inf.v0
+  version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4
+- name: gopkg.in/redis.v2
+  version: e6179049628164864e6e84e973cfb56335748dea
+devImports: []

weed/glide.yaml

@@ -0,0 +1,31 @@
+package: github.com/chrislusf/seaweedfs/weed
+import:
+- package: bazil.org/fuse
+  subpackages:
+  - fs
+- package: github.com/boltdb/bolt
+- package: github.com/chrislusf/raft
+- package: github.com/dgrijalva/jwt-go
+- package: github.com/disintegration/imaging
+- package: github.com/gocql/gocql
+- package: github.com/golang/protobuf
+  subpackages:
+  - proto
+- package: github.com/gorilla/mux
+- package: github.com/hashicorp/golang-lru
+- package: github.com/klauspost/crc32
+- package: github.com/reducedb/encoding
+  subpackages:
+  - cursor
+  - delta/bp32
+- package: github.com/rwcarlsen/goexif
+  subpackages:
+  - exif
+- package: github.com/syndtr/goleveldb
+  subpackages:
+  - leveldb
+  - leveldb/util
+- package: golang.org/x/net
+  subpackages:
+  - context
+- package: gopkg.in/redis.v2

weed/glog/glog.go

@@ -37,6 +37,7 @@
 //
 // By default, all log statements write to files in a temporary directory.
 // This package provides several flags that modify this behavior.
+// As a result, flag.Parse must be called before any logging is done.
 //
 //	-logtostderr=false
 //		Logs are written to standard error instead of to files.
@@ -45,6 +46,9 @@
 //	-stderrthreshold=ERROR
 //		Log events at or above this severity are logged to standard
 //		error as well as to files.
+//	-log_dir=""
+//		Log files will be written to this directory instead of the
+//		default temporary directory.
 //
 //	Other flags provide aids to debugging.
 //
@@ -73,6 +77,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	stdLog "log"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -89,6 +94,9 @@ import (
 // the corresponding constants in C++.
 type severity int32 // sync/atomic int32
 
+// These constants identify the log levels in order of increasing severity.
+// A message written to a high-severity log file is also written to each
+// lower-severity log file.
 const (
 	infoLog severity = iota
 	warningLog
@@ -307,7 +315,7 @@ func (m *moduleSpec) Set(value string) error {
 // isLiteral reports whether the pattern is a literal string, that is, has no metacharacters
 // that require filepath.Match to be called to match the pattern.
 func isLiteral(pattern string) bool {
-	return !strings.ContainsAny(pattern, `*?[]\`)
+	return !strings.ContainsAny(pattern, `\*?[]`)
 }
 
 // traceLocation represents the setting of the -log_backtrace_at flag.
@@ -446,8 +454,7 @@ type loggingT struct {
 	vmodule   moduleSpec // The state of the -vmodule flag.
 	verbosity Level      // V logging level, the value of the -v flag/
 
-	// If the logging exits, the system should still run.
-	// This flag is to disable the logging if logging exited.
+	// added by seaweedfs
 	exited bool
 }
 
@@ -466,7 +473,7 @@ func (l *loggingT) setVState(verbosity Level, filter []modulePat, setFilter bool
 	// Turn verbosity off so V will not fire while we are in transition.
 	logging.verbosity.set(0)
 	// Ditto for filter length.
-	logging.filterLength = 0
+	atomic.StoreInt32(&logging.filterLength, 0)
 
 	// Set the new filters and wipe the pc->Level map if the filter has changed.
 	if setFilter {
@@ -513,7 +520,8 @@ var timeNow = time.Now // Stubbed out for testing.
 
 /*
 header formats a log header as defined by the C++ implementation.
-It returns a buffer containing the formatted header.
+It returns a buffer containing the formatted header and the user's file and line number.
+The depth specifies how many stack frames above lives the source line to be identified in the log message.
 
 Log lines have this form:
 	Lmmdd hh:mm:ss.uuuuuu threadid file:line] msg...
@@ -527,10 +535,8 @@ where the fields are defined as follows:
 	line             The line number
 	msg              The user-supplied message
 */
-func (l *loggingT) header(s severity) *buffer {
-	// Lmmdd hh:mm:ss.uuuuuu threadid file:line]
-	now := timeNow()
-	_, file, line, ok := runtime.Caller(3) // It's always the same number of frames to the user's call.
+func (l *loggingT) header(s severity, depth int) (*buffer, string, int) {
+	_, file, line, ok := runtime.Caller(3 + depth)
 	if !ok {
 		file = "???"
 		line = 1
@@ -540,6 +546,12 @@ func (l *loggingT) header(s severity) *buffer {
 			file = file[slash+1:]
 		}
 	}
+	return l.formatHeader(s, file, line), file, line
+}
+
+// formatHeader formats a log header using the provided file name and line number.
+func (l *loggingT) formatHeader(s severity, file string, line int) *buffer {
+	now := timeNow()
 	if line < 0 {
 		line = 0 // not a real line number, but acceptable to someDigits
 	}
@@ -552,6 +564,7 @@ func (l *loggingT) header(s severity) *buffer {
 	// It's worth about 3X. Fprintf is hard.
 	_, month, day := now.Date()
 	hour, minute, second := now.Clock()
+	// Lmmdd hh:mm:ss.uuuuuu threadid file:line]
 	buf.tmp[0] = severityChar[s]
 	buf.twoDigits(1, int(month))
 	buf.twoDigits(3, day)
@@ -562,7 +575,7 @@ func (l *loggingT) header(s severity) *buffer {
 	buf.tmp[11] = ':'
 	buf.twoDigits(12, second)
 	buf.tmp[14] = ' '
-	buf.nDigits(5, 15, pid) // TODO: should be TID
+	buf.nDigits(5, 15, pid, ' ') // TODO: should be TID
 	buf.tmp[20] = ' '
 	buf.Write(buf.tmp[:21])
 	buf.WriteString(file)
@@ -585,12 +598,18 @@ func (buf *buffer) twoDigits(i, d int) {
 	buf.tmp[i] = digits[d%10]
 }
 
-// nDigits formats a zero-prefixed n-digit integer at buf.tmp[i].
-func (buf *buffer) nDigits(n, i, d int) {
-	for j := n - 1; j >= 0; j-- {
+// nDigits formats an n-digit integer at buf.tmp[i],
+// padding with pad on the left.
+// It assumes d >= 0.
+func (buf *buffer) nDigits(n, i, d int, pad byte) {
+	j := n - 1
+	for ; j >= 0 && d > 0; j-- {
 		buf.tmp[i+j] = digits[d%10]
 		d /= 10
 	}
+	for ; j >= 0; j-- {
+		buf.tmp[i+j] = pad
+	}
 }
 
 // someDigits formats a zero-prefixed variable-width integer at buf.tmp[i].
@@ -610,35 +629,50 @@ func (buf *buffer) someDigits(i, d int) int {
 }
 
 func (l *loggingT) println(s severity, args ...interface{}) {
-	buf := l.header(s)
+	buf, file, line := l.header(s, 0)
 	fmt.Fprintln(buf, args...)
-	l.output(s, buf)
+	l.output(s, buf, file, line, false)
 }
 
 func (l *loggingT) print(s severity, args ...interface{}) {
-	buf := l.header(s)
+	l.printDepth(s, 1, args...)
+}
+
+func (l *loggingT) printDepth(s severity, depth int, args ...interface{}) {
+	buf, file, line := l.header(s, depth)
 	fmt.Fprint(buf, args...)
 	if buf.Bytes()[buf.Len()-1] != '\n' {
 		buf.WriteByte('\n')
 	}
-	l.output(s, buf)
+	l.output(s, buf, file, line, false)
 }
 
 func (l *loggingT) printf(s severity, format string, args ...interface{}) {
-	buf := l.header(s)
+	buf, file, line := l.header(s, 0)
 	fmt.Fprintf(buf, format, args...)
 	if buf.Bytes()[buf.Len()-1] != '\n' {
 		buf.WriteByte('\n')
 	}
-	l.output(s, buf)
+	l.output(s, buf, file, line, false)
+}
+
+// printWithFileLine behaves like print but uses the provided file and line number.  If
+// alsoLogToStderr is true, the log message always appears on standard error; it
+// will also appear in the log file unless --logtostderr is set.
+func (l *loggingT) printWithFileLine(s severity, file string, line int, alsoToStderr bool, args ...interface{}) {
+	buf := l.formatHeader(s, file, line)
+	fmt.Fprint(buf, args...)
+	if buf.Bytes()[buf.Len()-1] != '\n' {
+		buf.WriteByte('\n')
+	}
+	l.output(s, buf, file, line, alsoToStderr)
 }
 
 // output writes the data to the log files and releases the buffer.
-func (l *loggingT) output(s severity, buf *buffer) {
+func (l *loggingT) output(s severity, buf *buffer, file string, line int, alsoToStderr bool) {
 	l.mu.Lock()
 	if l.traceLocation.isSet() {
-		_, file, line, ok := runtime.Caller(3) // It's always the same number of frames to the user's call (same as header).
-		if ok && l.traceLocation.match(file, line) {
+		if l.traceLocation.match(file, line) {
 			buf.Write(stacks(false))
 		}
 	}
@@ -646,7 +680,7 @@ func (l *loggingT) output(s severity, buf *buffer) {
 	if l.toStderr {
 		os.Stderr.Write(data)
 	} else {
-		if l.alsoToStderr || s >= l.stderrThreshold.get() {
+		if alsoToStderr || l.alsoToStderr || s >= l.stderrThreshold.get() {
 			os.Stderr.Write(data)
 		}
 		if l.file[s] == nil {
@@ -670,7 +704,16 @@ func (l *loggingT) output(s severity, buf *buffer) {
 		}
 	}
 	if s == fatalLog {
-		// Make sure we see the trace for the current goroutine on standard error.
+		// If we got here via Exit rather than Fatal, print no stacks.
+		if atomic.LoadUint32(&fatalNoStacks) > 0 {
+			l.mu.Unlock()
+			timeoutFlush(10 * time.Second)
+			os.Exit(1)
+		}
+		// Dump all goroutine stacks before exiting.
+		// First, make sure we see the trace for the current goroutine on standard error.
+		// If -logtostderr has been specified, the loop below will do that anyway
+		// as the first stack in the full dump.
 		if !l.toStderr {
 			os.Stderr.Write(stacks(false))
 		}
@@ -862,6 +905,54 @@ func (l *loggingT) flushAll() {
 	}
 }
 
+// CopyStandardLogTo arranges for messages written to the Go "log" package's
+// default logs to also appear in the Google logs for the named and lower
+// severities.  Subsequent changes to the standard log's default output location
+// or format may break this behavior.
+//
+// Valid names are "INFO", "WARNING", "ERROR", and "FATAL".  If the name is not
+// recognized, CopyStandardLogTo panics.
+func CopyStandardLogTo(name string) {
+	sev, ok := severityByName(name)
+	if !ok {
+		panic(fmt.Sprintf("log.CopyStandardLogTo(%q): unrecognized severity name", name))
+	}
+	// Set a log format that captures the user's file and line:
+	//   d.go:23: message
+	stdLog.SetFlags(stdLog.Lshortfile)
+	stdLog.SetOutput(logBridge(sev))
+}
+
+// logBridge provides the Write method that enables CopyStandardLogTo to connect
+// Go's standard logs to the logs provided by this package.
+type logBridge severity
+
+// Write parses the standard logging line and passes its components to the
+// logger for severity(lb).
+func (lb logBridge) Write(b []byte) (n int, err error) {
+	var (
+		file = "???"
+		line = 1
+		text string
+	)
+	// Split "d.go:23: message" into "d.go", "23", and "message".
+	if parts := bytes.SplitN(b, []byte{':'}, 3); len(parts) != 3 || len(parts[0]) < 1 || len(parts[2]) < 1 {
+		text = fmt.Sprintf("bad log format: %s", b)
+	} else {
+		file = string(parts[0])
+		text = string(parts[2][1:]) // skip leading space
+		line, err = strconv.Atoi(string(parts[1]))
+		if err != nil {
+			text = fmt.Sprintf("bad line number: %s", b)
+			line = 1
+		}
+	}
+	// printWithFileLine with alsoToStderr=true, so standard log messages
+	// always appear on standard error.
+	logging.printWithFileLine(severity(lb), file, line, true, text)
+	return len(b), nil
+}
+
 // setV computes and remembers the V level for a given PC
 // when vmodule is enabled.
 // File pattern matching takes the basename of the file, stripped
@@ -965,6 +1056,12 @@ func Info(args ...interface{}) {
 	logging.print(infoLog, args...)
 }
 
+// InfoDepth acts as Info but uses depth to determine which call frame to log.
+// InfoDepth(0, "msg") is the same as Info("msg").
+func InfoDepth(depth int, args ...interface{}) {
+	logging.printDepth(infoLog, depth, args...)
+}
+
 // Infoln logs to the INFO log.
 // Arguments are handled in the manner of fmt.Println; a newline is appended if missing.
 func Infoln(args ...interface{}) {
@@ -983,6 +1080,12 @@ func Warning(args ...interface{}) {
 	logging.print(warningLog, args...)
 }
 
+// WarningDepth acts as Warning but uses depth to determine which call frame to log.
+// WarningDepth(0, "msg") is the same as Warning("msg").
+func WarningDepth(depth int, args ...interface{}) {
+	logging.printDepth(warningLog, depth, args...)
+}
+
 // Warningln logs to the WARNING and INFO logs.
 // Arguments are handled in the manner of fmt.Println; a newline is appended if missing.
 func Warningln(args ...interface{}) {
@@ -1001,6 +1104,12 @@ func Error(args ...interface{}) {
 	logging.print(errorLog, args...)
 }
 
+// ErrorDepth acts as Error but uses depth to determine which call frame to log.
+// ErrorDepth(0, "msg") is the same as Error("msg").
+func ErrorDepth(depth int, args ...interface{}) {
+	logging.printDepth(errorLog, depth, args...)
+}
+
 // Errorln logs to the ERROR, WARNING, and INFO logs.
 // Arguments are handled in the manner of fmt.Println; a newline is appended if missing.
 func Errorln(args ...interface{}) {
@@ -1020,6 +1129,12 @@ func Fatal(args ...interface{}) {
 	logging.print(fatalLog, args...)
 }
 
+// FatalDepth acts as Fatal but uses depth to determine which call frame to log.
+// FatalDepth(0, "msg") is the same as Fatal("msg").
+func FatalDepth(depth int, args ...interface{}) {
+	logging.printDepth(fatalLog, depth, args...)
+}
+
 // Fatalln logs to the FATAL, ERROR, WARNING, and INFO logs,
 // including a stack trace of all running goroutines, then calls os.Exit(255).
 // Arguments are handled in the manner of fmt.Println; a newline is appended if missing.
@@ -1033,3 +1148,34 @@ func Fatalln(args ...interface{}) {
 func Fatalf(format string, args ...interface{}) {
 	logging.printf(fatalLog, format, args...)
 }
+
+// fatalNoStacks is non-zero if we are to exit without dumping goroutine stacks.
+// It allows Exit and relatives to use the Fatal logs.
+var fatalNoStacks uint32
+
+// Exit logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1).
+// Arguments are handled in the manner of fmt.Print; a newline is appended if missing.
+func Exit(args ...interface{}) {
+	atomic.StoreUint32(&fatalNoStacks, 1)
+	logging.print(fatalLog, args...)
+}
+
+// ExitDepth acts as Exit but uses depth to determine which call frame to log.
+// ExitDepth(0, "msg") is the same as Exit("msg").
+func ExitDepth(depth int, args ...interface{}) {
+	atomic.StoreUint32(&fatalNoStacks, 1)
+	logging.printDepth(fatalLog, depth, args...)
+}
+
+// Exitln logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1).
+func Exitln(args ...interface{}) {
+	atomic.StoreUint32(&fatalNoStacks, 1)
+	logging.println(fatalLog, args...)
+}
+
+// Exitf logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1).
+// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing.
+func Exitf(format string, args ...interface{}) {
+	atomic.StoreUint32(&fatalNoStacks, 1)
+	logging.printf(fatalLog, format, args...)
+}

weed/glog/glog_file.go

@@ -114,8 +114,8 @@ func create(tag string, t time.Time) (f *os.File, filename string, err error) {
 		f, err := os.Create(fname)
 		if err == nil {
 			symlink := filepath.Join(dir, link)
-			os.Remove(symlink)         // ignore err
-			os.Symlink(fname, symlink) // ignore err
+			os.Remove(symlink)        // ignore err
+			os.Symlink(name, symlink) // ignore err
 			return f, fname, nil
 		}
 		lastErr = err

weed/glog/glog_test.go

@@ -19,8 +19,10 @@ package glog
 import (
 	"bytes"
 	"fmt"
+	stdLog "log"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -96,20 +98,99 @@ func TestInfo(t *testing.T) {
 	}
 }
 
+func TestInfoDepth(t *testing.T) {
+	setFlags()
+	defer logging.swap(logging.newBuffers())
+
+	f := func() { InfoDepth(1, "depth-test1") }
+
+	// The next three lines must stay together
+	_, _, wantLine, _ := runtime.Caller(0)
+	InfoDepth(0, "depth-test0")
+	f()
+
+	msgs := strings.Split(strings.TrimSuffix(contents(infoLog), "\n"), "\n")
+	if len(msgs) != 2 {
+		t.Fatalf("Got %d lines, expected 2", len(msgs))
+	}
+
+	for i, m := range msgs {
+		if !strings.HasPrefix(m, "I") {
+			t.Errorf("InfoDepth[%d] has wrong character: %q", i, m)
+		}
+		w := fmt.Sprintf("depth-test%d", i)
+		if !strings.Contains(m, w) {
+			t.Errorf("InfoDepth[%d] missing %q: %q", i, w, m)
+		}
+
+		// pull out the line number (between : and ])
+		msg := m[strings.LastIndex(m, ":")+1:]
+		x := strings.Index(msg, "]")
+		if x < 0 {
+			t.Errorf("InfoDepth[%d]: missing ']': %q", i, m)
+			continue
+		}
+		line, err := strconv.Atoi(msg[:x])
+		if err != nil {
+			t.Errorf("InfoDepth[%d]: bad line number: %q", i, m)
+			continue
+		}
+		wantLine++
+		if wantLine != line {
+			t.Errorf("InfoDepth[%d]: got line %d, want %d", i, line, wantLine)
+		}
+	}
+}
+
+func init() {
+	CopyStandardLogTo("INFO")
+}
+
+// Test that CopyStandardLogTo panics on bad input.
+func TestCopyStandardLogToPanic(t *testing.T) {
+	defer func() {
+		if s, ok := recover().(string); !ok || !strings.Contains(s, "LOG") {
+			t.Errorf(`CopyStandardLogTo("LOG") should have panicked: %v`, s)
+		}
+	}()
+	CopyStandardLogTo("LOG")
+}
+
+// Test that using the standard log package logs to INFO.
+func TestStandardLog(t *testing.T) {
+	setFlags()
+	defer logging.swap(logging.newBuffers())
+	stdLog.Print("test")
+	if !contains(infoLog, "I", t) {
+		t.Errorf("Info has wrong character: %q", contents(infoLog))
+	}
+	if !contains(infoLog, "test", t) {
+		t.Error("Info failed")
+	}
+}
+
 // Test that the header has the correct format.
 func TestHeader(t *testing.T) {
 	setFlags()
 	defer logging.swap(logging.newBuffers())
 	defer func(previous func() time.Time) { timeNow = previous }(timeNow)
 	timeNow = func() time.Time {
-		return time.Date(2006, 1, 2, 15, 4, 5, .678901e9, time.Local)
+		return time.Date(2006, 1, 2, 15, 4, 5, .067890e9, time.Local)
 	}
+	pid = 1234
 	Info("test")
-	var line, pid int
-	n, err := fmt.Sscanf(contents(infoLog), "I0102 15:04:05 %d glog_test.go:%d] test\n", &pid, &line)
-	if n != 2 || err != nil {
+	var line int
+	format := "I0102 15:04:05  1234 glog_test.go:%d] test\n"
+	n, err := fmt.Sscanf(contents(infoLog), format, &line)
+	if n != 1 || err != nil {
 		t.Errorf("log format error: %d elements, error %s:\n%s", n, err, contents(infoLog))
 	}
+	// Scanf treats multiple spaces as equivalent to a single space,
+	// so check for correct space-padding also.
+	want := fmt.Sprintf(format, line)
+	if contents(infoLog) != want {
+		t.Errorf("log format error: got:\n\t%q\nwant:\t%q", contents(infoLog), want)
+	}
 }
 
 // Test that an Error log goes to Warning and Info.
@@ -328,6 +409,7 @@ func TestLogBacktraceAt(t *testing.T) {
 
 func BenchmarkHeader(b *testing.B) {
 	for i := 0; i < b.N; i++ {
-		logging.putBuffer(logging.header(infoLog))
+		buf, _, _ := logging.header(infoLog, 0)
+		logging.putBuffer(buf)
 	}
 }

weed/images/favicon.go

@@ -0,0 +1,236 @@
+// Code generated by go-bindata.
+// sources:
+// favicon/favicon.ico
+
+package images
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func bindataRead(data []byte, name string) ([]byte, error) {
+	gz, err := gzip.NewReader(bytes.NewBuffer(data))
+	if err != nil {
+		return nil, fmt.Errorf("Read %q: %v", name, err)
+	}
+
+	var buf bytes.Buffer
+	_, err = io.Copy(&buf, gz)
+	clErr := gz.Close()
+
+	if err != nil {
+		return nil, fmt.Errorf("Read %q: %v", name, err)
+	}
+	if clErr != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+type asset struct {
+	bytes []byte
+	info  os.FileInfo
+}
+
+type bindataFileInfo struct {
+	name    string
+	size    int64
+	mode    os.FileMode
+	modTime time.Time
+}
+
+func (fi bindataFileInfo) Name() string {
+	return fi.name
+}
+func (fi bindataFileInfo) Size() int64 {
+	return fi.size
+}
+func (fi bindataFileInfo) Mode() os.FileMode {
+	return fi.mode
+}
+func (fi bindataFileInfo) ModTime() time.Time {
+	return fi.modTime
+}
+func (fi bindataFileInfo) IsDir() bool {
+	return false
+}
+func (fi bindataFileInfo) Sys() interface{} {
+	return nil
+}
+
+var _favicon = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xec\x94\x5f\x48\x53\x51\x1c\xc7\xbf\x77\xce\xeb\xf1\xff\x9d\x53\x37\xe7\xbf\x4d\xcd\x2c\x08\xa2\x87\xea\xd1\x82\x9e\xa2\x88\x5e\x7a\x8a\x82\x1e\xf2\x21\x7a\x28\x42\xc9\x7a\xb0\x08\x22\x4d\x08\xfa\x43\x25\x61\x05\x4b\xac\xb7\x7a\x92\x8a\x82\xa2\x34\x59\x5c\x4d\x5b\x39\x2d\xfb\xb3\x9c\xe0\x52\xb7\x5a\x6e\x73\xeb\xdc\xbb\xed\xb6\x5d\xc7\x89\x7a\xa8\xa0\xfb\x81\x7b\xef\x39\xbf\xef\x3d\xf7\xfe\xce\x79\xf8\x00\x1c\x74\x10\x04\xe9\x49\xd0\x94\x09\xd4\x03\xb0\x5a\x63\xf3\x1e\x02\x74\xd2\x5a\x03\xad\x09\x52\x1d\xb1\xba\x0c\x87\x1f\x70\xb1\xab\xff\xd0\x2a\x0c\x9d\xde\x0e\xf1\xe4\x66\xbc\xba\xb2\x0f\xe1\xc0\x3c\x5e\x77\xef\x87\xe7\xe9\x4d\xcc\x3a\x1f\x21\x1a\x8d\x22\x1a\x89\xc0\x7d\xbf\x0b\x03\xcd\xab\x31\xde\x73\x18\x6f\x6e\xb5\x61\xea\xf1\x0d\x84\xbf\xf9\x31\x76\xfd\x20\xbe\x4e\xb9\x20\xb6\x6f\xc5\xb3\xd6\xb5\x78\x7e\x62\x13\x7d\xae\x83\xa3\xad\x11\x73\xae\x7e\xbc\x38\xb3\x03\x23\x67\x77\x62\x61\xd6\x03\x67\x57\x13\x1c\xc7\x37\x62\xa2\xf7\x28\x22\xe1\x10\x7c\x93\x22\x42\x7e\x2f\x82\x73\xd3\x18\x68\x59\x03\x97\xbd\x05\x43\x1d\xdb\xa0\xa1\xa1\xf1\xaf\x51\x5b\x97\x3c\xab\x29\x20\xe6\xa4\x69\x71\x35\x21\x56\x1b\x25\x3e\xaf\xac\x22\x31\x12\x2f\x94\x59\x48\x79\x05\x45\x7e\xb9\x84\xde\x4a\x4d\xca\xfa\x0c\x43\x11\xbd\xe7\x1a\xa5\x71\x3e\xa5\x80\x14\x0a\x89\x2c\xfe\x19\x92\x17\x9f\xf3\x94\x2c\x92\x9d\x93\x9b\xf4\xf3\x0c\x7d\x66\x4a\xa7\x9c\xee\xb7\x37\xf9\xcb\x48\x9e\xb5\xd2\xab\x11\x49\x9e\xd5\x27\x37\x03\xc5\xb3\x8e\x63\x1b\xe0\x79\xd2\x8b\x4f\x0f\xaf\xca\x2e\x95\x1c\x2b\xf9\x75\x7e\x7c\x50\x76\xe0\xe8\xf9\x5d\xb2\x2f\xbd\xc3\x77\x11\x59\x0c\x61\xf2\x76\x3b\x66\xc4\x3e\x0c\x1e\x59\x8f\xc0\xf4\x84\xec\xd6\x0f\x7d\xe7\xe0\x7f\x3f\x22\x7b\x33\xf4\xe5\xb3\xec\x54\xc9\x9d\x8b\xc1\x00\x75\x73\x2b\x86\xa9\xcb\xdf\xdd\xe9\x94\x1d\xed\xb2\x37\x53\x4f\xdb\xe1\x7b\x2b\xe2\xe3\xbd\x4b\x78\x79\x71\x0f\x82\xbe\x19\x38\x2f\xef\xa5\xdf\xee\xc0\xe8\x85\xdd\x58\xf0\xba\xe1\x7e\xd0\x8d\xb1\x6b\x07\x64\x9f\x8b\xa7\xb6\xfc\xb9\xc3\xd3\xd0\xd0\xf8\x7b\xac\xb4\x31\xe3\x86\x15\x15\x8c\x54\xbf\x9c\xe7\x19\x39\x97\xc5\xb3\xf2\x65\x44\x32\x79\xbd\x31\x81\x2a\xae\xcb\xe3\x53\x49\x8d\x6b\x79\x35\xaa\xf5\xb6\x1a\x76\x0e\x43\xb5\x54\xb5\xe6\x24\x58\xd2\xa0\xa9\x8a\xd9\x3f\xa5\xd2\x9c\x2e\x2f\xb7\x28\x43\x8b\x90\x26\xcf\x34\x97\x29\xe3\x42\x4b\x9a\x9c\xe7\x75\x26\xc6\x5f\x69\xce\x1b\x0d\x4b\xca\xca\x41\x16\x4b\xfb\x2a\x29\x55\xe7\xea\xf3\x2a\xfa\x49\x2e\x18\x98\x79\x41\xa1\x6a\xbd\x72\x90\xb9\x34\xcd\xcb\x67\xf6\x4f\xb2\x99\xfb\xcb\xe2\x18\x31\x74\x19\x7a\x56\x0c\xe6\xe2\xff\x97\xef\x01\x00\x00\xff\xff\x3e\xd4\x17\xe7\x36\x0e\x00\x00")
+
+func faviconBytes() ([]byte, error) {
+	return bindataRead(
+		_favicon,
+		"favicon/favicon.ico",
+	)
+}
+
+func favicon() (*asset, error) {
+	bytes, err := faviconBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "favicon/favicon.ico", size: 3638, mode: os.FileMode(420), modTime: time.Unix(1460621502, 0)}
+	a := &asset{bytes: bytes, info: info}
+	return a, nil
+}
+
+// Asset loads and returns the asset for the given name.
+// It returns an error if the asset could not be found or
+// could not be loaded.
+func Asset(name string) ([]byte, error) {
+	cannonicalName := strings.Replace(name, "\\", "/", -1)
+	if f, ok := _bindata[cannonicalName]; ok {
+		a, err := f()
+		if err != nil {
+			return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
+		}
+		return a.bytes, nil
+	}
+	return nil, fmt.Errorf("Asset %s not found", name)
+}
+
+// MustAsset is like Asset but panics when Asset would return an error.
+// It simplifies safe initialization of global variables.
+func MustAsset(name string) []byte {
+	a, err := Asset(name)
+	if err != nil {
+		panic("asset: Asset(" + name + "): " + err.Error())
+	}
+
+	return a
+}
+
+// AssetInfo loads and returns the asset info for the given name.
+// It returns an error if the asset could not be found or
+// could not be loaded.
+func AssetInfo(name string) (os.FileInfo, error) {
+	cannonicalName := strings.Replace(name, "\\", "/", -1)
+	if f, ok := _bindata[cannonicalName]; ok {
+		a, err := f()
+		if err != nil {
+			return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
+		}
+		return a.info, nil
+	}
+	return nil, fmt.Errorf("AssetInfo %s not found", name)
+}
+
+// AssetNames returns the names of the assets.
+func AssetNames() []string {
+	names := make([]string, 0, len(_bindata))
+	for name := range _bindata {
+		names = append(names, name)
+	}
+	return names
+}
+
+// _bindata is a table, holding each asset generator, mapped to its name.
+var _bindata = map[string]func() (*asset, error){
+	"favicon/favicon.ico": favicon,
+}
+
+// AssetDir returns the file names below a certain
+// directory embedded in the file by go-bindata.
+// For example if you run go-bindata on data/... and data contains the
+// following hierarchy:
+//     data/
+//       foo.txt
+//       img/
+//         a.png
+//         b.png
+// then AssetDir("data") would return []string{"foo.txt", "img"}
+// AssetDir("data/img") would return []string{"a.png", "b.png"}
+// AssetDir("foo.txt") and AssetDir("notexist") would return an error
+// AssetDir("") will return []string{"data"}.
+func AssetDir(name string) ([]string, error) {
+	node := _bintree
+	if len(name) != 0 {
+		cannonicalName := strings.Replace(name, "\\", "/", -1)
+		pathList := strings.Split(cannonicalName, "/")
+		for _, p := range pathList {
+			node = node.Children[p]
+			if node == nil {
+				return nil, fmt.Errorf("Asset %s not found", name)
+			}
+		}
+	}
+	if node.Func != nil {
+		return nil, fmt.Errorf("Asset %s not found", name)
+	}
+	rv := make([]string, 0, len(node.Children))
+	for childName := range node.Children {
+		rv = append(rv, childName)
+	}
+	return rv, nil
+}
+
+type bintree struct {
+	Func     func() (*asset, error)
+	Children map[string]*bintree
+}
+var _bintree = &bintree{nil, map[string]*bintree{
+	"favicon": &bintree{nil, map[string]*bintree{
+		"favicon.ico": &bintree{favicon, map[string]*bintree{}},
+	}},
+}}
+
+// RestoreAsset restores an asset under the given directory
+func RestoreAsset(dir, name string) error {
+	data, err := Asset(name)
+	if err != nil {
+		return err
+	}
+	info, err := AssetInfo(name)
+	if err != nil {
+		return err
+	}
+	err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
+	if err != nil {
+		return err
+	}
+	err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// RestoreAssets restores an asset under the given directory recursively
+func RestoreAssets(dir, name string) error {
+	children, err := AssetDir(name)
+	// File
+	if err != nil {
+		return RestoreAsset(dir, name)
+	}
+	// Dir
+	for _, child := range children {
+		err = RestoreAssets(dir, filepath.Join(name, child))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func _filePath(dir, name string) string {
+	cannonicalName := strings.Replace(name, "\\", "/", -1)
+	return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
+}
+

weed/images/orientation_test.go

@@ -2,6 +2,7 @@ package images
 
 import (
 	"io/ioutil"
+	"os"
 	"testing"
 )
 
@@ -14,4 +15,6 @@ func TestXYZ(t *testing.T) {
 
 	ioutil.WriteFile("fixed1.jpg", fixed_data, 0644)
 
+	os.Remove("fixed1.jpg")
+
 }

weed/images/resizing.go

@@ -7,7 +7,7 @@ import (
 	"image/jpeg"
 	"image/png"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/disintegration/imaging"
 )
 

weed/operation/assign_file_id.go

@@ -7,10 +7,20 @@ import (
 	"net/url"
 	"strconv"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/util"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
+type VolumeAssignRequest struct {
+	Count       uint64
+	Replication string
+	Collection  string
+	Ttl         string
+	DataCenter  string
+	Rack        string
+	DataNode    string
+}
+
 type AssignResult struct {
 	Fid       string `json:"fid,omitempty"`
 	Url       string `json:"url,omitempty"`
@@ -19,18 +29,28 @@ type AssignResult struct {
 	Error     string `json:"error,omitempty"`
 }
 
-func Assign(server string, count uint64, replication string, collection string, ttl string) (*AssignResult, error) {
+func Assign(server string, r *VolumeAssignRequest) (*AssignResult, error) {
 	values := make(url.Values)
-	values.Add("count", strconv.FormatUint(count, 10))
-	if replication != "" {
-		values.Add("replication", replication)
+	values.Add("count", strconv.FormatUint(r.Count, 10))
+	if r.Replication != "" {
+		values.Add("replication", r.Replication)
 	}
-	if collection != "" {
-		values.Add("collection", collection)
+	if r.Collection != "" {
+		values.Add("collection", r.Collection)
 	}
-	if ttl != "" {
-		values.Add("ttl", ttl)
+	if r.Ttl != "" {
+		values.Add("ttl", r.Ttl)
 	}
+	if r.DataCenter != "" {
+		values.Add("dataCenter", r.DataCenter)
+	}
+	if r.Rack != "" {
+		values.Add("rack", r.Rack)
+	}
+	if r.DataNode != "" {
+		values.Add("dataNode", r.DataNode)
+	}
+
 	jsonBlob, err := util.Post("http://"+server+"/dir/assign", values)
 	glog.V(2).Info("assign result :", string(jsonBlob))
 	if err != nil {

weed/operation/chunked_file.go

@@ -0,0 +1,213 @@
+package operation
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"sort"
+
+	"sync"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+var (
+	// when the remote server does not allow range requests (Accept-Ranges was not set)
+	ErrRangeRequestsNotSupported = errors.New("Range requests are not supported by the remote server")
+	// ErrInvalidRange is returned by Read when trying to read past the end of the file
+	ErrInvalidRange = errors.New("Invalid range")
+)
+
+type ChunkInfo struct {
+	Fid    string `json:"fid"`
+	Offset int64  `json:"offset"`
+	Size   int64  `json:"size"`
+}
+
+type ChunkList []*ChunkInfo
+
+type ChunkManifest struct {
+	Name   string    `json:"name,omitempty"`
+	Mime   string    `json:"mime,omitempty"`
+	Size   int64     `json:"size,omitempty"`
+	Chunks ChunkList `json:"chunks,omitempty"`
+}
+
+// seekable chunked file reader
+type ChunkedFileReader struct {
+	Manifest *ChunkManifest
+	Master   string
+	pos      int64
+	pr       *io.PipeReader
+	pw       *io.PipeWriter
+	mutex    sync.Mutex
+}
+
+func (s ChunkList) Len() int           { return len(s) }
+func (s ChunkList) Less(i, j int) bool { return s[i].Offset < s[j].Offset }
+func (s ChunkList) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
+func LoadChunkManifest(buffer []byte, isGzipped bool) (*ChunkManifest, error) {
+	if isGzipped {
+		var err error
+		if buffer, err = UnGzipData(buffer); err != nil {
+			return nil, err
+		}
+	}
+	cm := ChunkManifest{}
+	if e := json.Unmarshal(buffer, &cm); e != nil {
+		return nil, e
+	}
+	sort.Sort(cm.Chunks)
+	return &cm, nil
+}
+
+func (cm *ChunkManifest) Marshal() ([]byte, error) {
+	return json.Marshal(cm)
+}
+
+func (cm *ChunkManifest) DeleteChunks(master string) error {
+	deleteError := 0
+	for _, ci := range cm.Chunks {
+		if e := DeleteFile(master, ci.Fid, ""); e != nil {
+			deleteError++
+			glog.V(0).Infof("Delete %s error: %v, master: %s", ci.Fid, e, master)
+		}
+	}
+	if deleteError > 0 {
+		return errors.New("Not all chunks deleted.")
+	}
+	return nil
+}
+
+func readChunkNeedle(fileUrl string, w io.Writer, offset int64) (written int64, e error) {
+	req, err := http.NewRequest("GET", fileUrl, nil)
+	if err != nil {
+		return written, err
+	}
+	if offset > 0 {
+		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
+	}
+
+	resp, err := util.Do(req)
+	if err != nil {
+		return written, err
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusRequestedRangeNotSatisfiable:
+		return written, ErrInvalidRange
+	case http.StatusOK:
+		if offset > 0 {
+			return written, ErrRangeRequestsNotSupported
+		}
+	case http.StatusPartialContent:
+		break
+	default:
+		return written, fmt.Errorf("Read chunk needle error: [%d] %s", resp.StatusCode, fileUrl)
+
+	}
+	return io.Copy(w, resp.Body)
+}
+
+func (cf *ChunkedFileReader) Seek(offset int64, whence int) (int64, error) {
+	var err error
+	switch whence {
+	case 0:
+	case 1:
+		offset += cf.pos
+	case 2:
+		offset = cf.Manifest.Size - offset
+	}
+	if offset > cf.Manifest.Size {
+		err = ErrInvalidRange
+	}
+	if cf.pos != offset {
+		cf.Close()
+	}
+	cf.pos = offset
+	return cf.pos, err
+}
+
+func (cf *ChunkedFileReader) WriteTo(w io.Writer) (n int64, err error) {
+	cm := cf.Manifest
+	chunkIndex := -1
+	chunkStartOffset := int64(0)
+	for i, ci := range cm.Chunks {
+		if cf.pos >= ci.Offset && cf.pos < ci.Offset+ci.Size {
+			chunkIndex = i
+			chunkStartOffset = cf.pos - ci.Offset
+			break
+		}
+	}
+	if chunkIndex < 0 {
+		return n, ErrInvalidRange
+	}
+	for ; chunkIndex < cm.Chunks.Len(); chunkIndex++ {
+		ci := cm.Chunks[chunkIndex]
+		// if we need read date from local volume server first?
+		fileUrl, lookupError := LookupFileId(cf.Master, ci.Fid)
+		if lookupError != nil {
+			return n, lookupError
+		}
+		if wn, e := readChunkNeedle(fileUrl, w, chunkStartOffset); e != nil {
+			return n, e
+		} else {
+			n += wn
+			cf.pos += wn
+		}
+
+		chunkStartOffset = 0
+	}
+	return n, nil
+}
+
+func (cf *ChunkedFileReader) ReadAt(p []byte, off int64) (n int, err error) {
+	cf.Seek(off, 0)
+	return cf.Read(p)
+}
+
+func (cf *ChunkedFileReader) Read(p []byte) (int, error) {
+	return cf.getPipeReader().Read(p)
+}
+
+func (cf *ChunkedFileReader) Close() (e error) {
+	cf.mutex.Lock()
+	defer cf.mutex.Unlock()
+	return cf.closePipe()
+}
+
+func (cf *ChunkedFileReader) closePipe() (e error) {
+	if cf.pr != nil {
+		if err := cf.pr.Close(); err != nil {
+			e = err
+		}
+	}
+	cf.pr = nil
+	if cf.pw != nil {
+		if err := cf.pw.Close(); err != nil {
+			e = err
+		}
+	}
+	cf.pw = nil
+	return e
+}
+
+func (cf *ChunkedFileReader) getPipeReader() io.Reader {
+	cf.mutex.Lock()
+	defer cf.mutex.Unlock()
+	if cf.pr != nil && cf.pw != nil {
+		return cf.pr
+	}
+	cf.closePipe()
+	cf.pr, cf.pw = io.Pipe()
+	go func(pw *io.PipeWriter) {
+		_, e := cf.WriteTo(pw)
+		pw.CloseWithError(e)
+	}(cf.pw)
+	return cf.pr
+}

weed/operation/compress.go

@@ -1,4 +1,4 @@
-package storage
+package operation
 
 import (
 	"bytes"
@@ -7,7 +7,7 @@ import (
 	"io/ioutil"
 	"strings"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 )
 
 /*

weed/operation/delete_content.go

@@ -3,26 +3,34 @@ package operation
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/url"
 	"strings"
 	"sync"
 
-	"github.com/chrislusf/seaweedfs/go/security"
-	"github.com/chrislusf/seaweedfs/go/util"
+	"net/http"
+
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 type DeleteResult struct {
-	Fid   string `json:"fid"`
-	Size  int    `json:"size"`
-	Error string `json:"error,omitempty"`
+	Fid    string `json:"fid"`
+	Size   int    `json:"size"`
+	Status int    `json:"status"`
+	Error  string `json:"error,omitempty"`
 }
 
 func DeleteFile(master string, fileId string, jwt security.EncodedJwt) error {
 	fileUrl, err := LookupFileId(master, fileId)
 	if err != nil {
-		return err
+		return fmt.Errorf("Failed to lookup %s:%v", fileId, err)
 	}
-	return util.Delete(fileUrl, jwt)
+	err = util.Delete(fileUrl, jwt)
+	if err != nil {
+		return fmt.Errorf("Failed to delete %s:%v", fileUrl, err)
+	}
+	return nil
 }
 
 func ParseFileId(fid string) (vid string, key_cookie string, err error) {
@@ -45,7 +53,11 @@ func DeleteFiles(master string, fileIds []string) (*DeleteFilesResult, error) {
 	for _, fileId := range fileIds {
 		vid, _, err := ParseFileId(fileId)
 		if err != nil {
-			ret.Results = append(ret.Results, DeleteResult{Fid: vid, Error: err.Error()})
+			ret.Results = append(ret.Results, DeleteResult{
+				Fid:    vid,
+				Status: http.StatusBadRequest,
+				Error:  err.Error()},
+			)
 			continue
 		}
 		if _, ok := vid_to_fileIds[vid]; !ok {
@@ -76,6 +88,7 @@ func DeleteFiles(master string, fileIds []string) (*DeleteFilesResult, error) {
 	}
 
 	var wg sync.WaitGroup
+
 	for server, fidList := range server_to_fileIds {
 		wg.Add(1)
 		go func(server string, fidList []string) {

weed/operation/filer/register.go

@@ -0,0 +1,31 @@
+package filer
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+type SubmitResult struct {
+	FileName string `json:"fileName,omitempty"`
+	FileUrl  string `json:"fileUrl,omitempty"`
+	Fid      string `json:"fid,omitempty"`
+	Size     uint32 `json:"size,omitempty"`
+	Error    string `json:"error,omitempty"`
+}
+
+func RegisterFile(filer string, path string, fileId string, secret security.Secret) error {
+	// TODO: jwt need to be used
+	_ = security.GenJwt(secret, fileId)
+
+	values := make(url.Values)
+	values.Add("path", path)
+	values.Add("fileId", fileId)
+	_, err := util.Post("http://"+filer+"/admin/register", values)
+	if err != nil {
+		return fmt.Errorf("Failed to register path:%s on filer:%s to file id:%s", path, filer, fileId)
+	}
+	return nil
+}

weed/operation/list_masters.go

@@ -3,8 +3,8 @@ package operation
 import (
 	"encoding/json"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/util"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 type ClusterStatusResult struct {
@@ -13,20 +13,20 @@ type ClusterStatusResult struct {
 	Peers    []string `json:"Peers,omitempty"`
 }
 
-func ListMasters(server string) ([]string, error) {
+func ListMasters(server string) (leader string, peers []string, err error) {
 	jsonBlob, err := util.Get("http://" + server + "/cluster/status")
 	glog.V(2).Info("list masters result :", string(jsonBlob))
 	if err != nil {
-		return nil, err
+		return "", nil, err
 	}
 	var ret ClusterStatusResult
 	err = json.Unmarshal(jsonBlob, &ret)
 	if err != nil {
-		return nil, err
+		return "", nil, err
 	}
-	masters := ret.Peers
+	peers = ret.Peers
 	if ret.IsLeader {
-		masters = append(masters, ret.Leader)
+		peers = append(peers, ret.Leader)
 	}
-	return masters, nil
+	return ret.Leader, peers, nil
 }

weed/operation/lookup.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/chrislusf/seaweedfs/go/util"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 type Location struct {

weed/operation/lookup_vid_cache.go

@@ -3,7 +3,10 @@ package operation
 import (
 	"errors"
 	"strconv"
+	"sync"
 	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
 )
 
 type VidInfo struct {
@@ -11,11 +14,18 @@ type VidInfo struct {
 	NextRefreshTime time.Time
 }
 type VidCache struct {
+	sync.RWMutex
 	cache []VidInfo
 }
 
 func (vc *VidCache) Get(vid string) ([]Location, error) {
-	id, _ := strconv.Atoi(vid)
+	id, err := strconv.Atoi(vid)
+	if err != nil {
+		glog.V(1).Infof("Unknown volume id %s", vid)
+		return nil, err
+	}
+	vc.RLock()
+	defer vc.RUnlock()
 	if 0 < id && id <= len(vc.cache) {
 		if vc.cache[id-1].Locations == nil {
 			return nil, errors.New("Not Set")
@@ -28,14 +38,20 @@ func (vc *VidCache) Get(vid string) ([]Location, error) {
 	return nil, errors.New("Not Found")
 }
 func (vc *VidCache) Set(vid string, locations []Location, duration time.Duration) {
-	id, _ := strconv.Atoi(vid)
-	if id >= len(vc.cache) {
+	id, err := strconv.Atoi(vid)
+	if err != nil {
+		glog.V(1).Infof("Unknown volume id %s", vid)
+		return
+	}
+	vc.Lock()
+	defer vc.Unlock()
+	if id > len(vc.cache) {
 		for i := id - len(vc.cache); i > 0; i-- {
 			vc.cache = append(vc.cache, VidInfo{})
 		}
 	}
-
-	vc.cache[id-1].Locations = locations
-	vc.cache[id-1].NextRefreshTime = time.Now().Add(duration)
-
+	if id > 0 {
+		vc.cache[id-1].Locations = locations
+		vc.cache[id-1].NextRefreshTime = time.Now().Add(duration)
+	}
 }

weed/operation/submit.go

@@ -4,13 +4,14 @@ import (
 	"bytes"
 	"io"
 	"mime"
+	"net/url"
 	"os"
 	"path"
 	"strconv"
 	"strings"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/security"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/security"
 )
 
 type FilePart struct {
@@ -22,6 +23,7 @@ type FilePart struct {
 	ModTime     int64 //in seconds
 	Replication string
 	Collection  string
+	DataCenter  string
 	Ttl         string
 	Server      string //this comes from assign result
 	Fid         string //this comes from assign result, but customizable
@@ -36,14 +38,21 @@ type SubmitResult struct {
 }
 
 func SubmitFiles(master string, files []FilePart,
-	replication string, collection string, ttl string, maxMB int,
+	replication string, collection string, dataCenter string, ttl string, maxMB int,
 	secret security.Secret,
 ) ([]SubmitResult, error) {
 	results := make([]SubmitResult, len(files))
 	for index, file := range files {
 		results[index].FileName = file.FileName
 	}
-	ret, err := Assign(master, uint64(len(files)), replication, collection, ttl)
+	ar := &VolumeAssignRequest{
+		Count:       uint64(len(files)),
+		Replication: replication,
+		Collection:  collection,
+		DataCenter:  dataCenter,
+		Ttl:         ttl,
+	}
+	ret, err := Assign(master, ar)
 	if err != nil {
 		for index, _ := range files {
 			results[index].Error = err.Error()
@@ -58,6 +67,7 @@ func SubmitFiles(master string, files []FilePart,
 		file.Server = ret.Url
 		file.Replication = replication
 		file.Collection = collection
+		file.DataCenter = dataCenter
 		results[index].Size, err = file.Upload(maxMB, master, secret)
 		if err != nil {
 			results[index].Error = err.Error()
@@ -85,18 +95,15 @@ func newFilePart(fullPathFilename string) (ret FilePart, err error) {
 	}
 	ret.Reader = fh
 
-	if fi, fiErr := fh.Stat(); fiErr != nil {
+	fi, fiErr := fh.Stat()
+	if fiErr != nil {
 		glog.V(0).Info("Failed to stat file:", fullPathFilename)
 		return ret, fiErr
-	} else {
-		ret.ModTime = fi.ModTime().UTC().Unix()
-		ret.FileSize = fi.Size()
 	}
+	ret.ModTime = fi.ModTime().UTC().Unix()
+	ret.FileSize = fi.Size()
 	ext := strings.ToLower(path.Ext(fullPathFilename))
 	ret.IsGzipped = ext == ".gz"
-	if ret.IsGzipped {
-		ret.FileName = fullPathFilename[0 : len(fullPathFilename)-3]
-	}
 	ret.FileName = fullPathFilename
 	if ext != "" {
 		ret.MimeType = mime.TypeByExtension(ext)
@@ -114,25 +121,79 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 	if closer, ok := fi.Reader.(io.Closer); ok {
 		defer closer.Close()
 	}
+	baseName := path.Base(fi.FileName)
 	if maxMB > 0 && fi.FileSize > int64(maxMB*1024*1024) {
 		chunkSize := int64(maxMB * 1024 * 1024)
 		chunks := fi.FileSize/chunkSize + 1
-		var fids []string
+		cm := ChunkManifest{
+			Name:   baseName,
+			Size:   fi.FileSize,
+			Mime:   fi.MimeType,
+			Chunks: make([]*ChunkInfo, 0, chunks),
+		}
+
+		var ret *AssignResult
+		var id string
+		if fi.DataCenter != "" {
+			ar := &VolumeAssignRequest{
+				Count:       uint64(chunks),
+				Replication: fi.Replication,
+				Collection:  fi.Collection,
+				Ttl:         fi.Ttl,
+			}
+			ret, err = Assign(master, ar)
+			if err != nil {
+				return
+			}
+		}
 		for i := int64(0); i < chunks; i++ {
-			id, count, e := upload_one_chunk(
-				fi.FileName+"-"+strconv.FormatInt(i+1, 10),
+			if fi.DataCenter == "" {
+				ar := &VolumeAssignRequest{
+					Count:       1,
+					Replication: fi.Replication,
+					Collection:  fi.Collection,
+					Ttl:         fi.Ttl,
+				}
+				ret, err = Assign(master, ar)
+				if err != nil {
+					// delete all uploaded chunks
+					cm.DeleteChunks(master)
+					return
+				}
+				id = ret.Fid
+			} else {
+				id = ret.Fid
+				if i > 0 {
+					id += "_" + strconv.FormatInt(i, 10)
+				}
+			}
+			fileUrl := "http://" + ret.Url + "/" + id
+			count, e := upload_one_chunk(
+				baseName+"-"+strconv.FormatInt(i+1, 10),
 				io.LimitReader(fi.Reader, chunkSize),
-				master, fi.Replication, fi.Collection, fi.Ttl,
+				master, fileUrl,
 				jwt)
 			if e != nil {
+				// delete all uploaded chunks
+				cm.DeleteChunks(master)
 				return 0, e
 			}
-			fids = append(fids, id)
+			cm.Chunks = append(cm.Chunks,
+				&ChunkInfo{
+					Offset: i * chunkSize,
+					Size:   int64(count),
+					Fid:    id,
+				},
+			)
 			retSize += count
 		}
-		err = upload_file_id_list(fileUrl, fi.FileName+"-list", fids, jwt)
+		err = upload_chunked_file_manifest(fileUrl, &cm, jwt)
+		if err != nil {
+			// delete all uploaded chunks
+			cm.DeleteChunks(master)
+		}
 	} else {
-		ret, e := Upload(fileUrl, fi.FileName, fi.Reader, fi.IsGzipped, fi.MimeType, jwt)
+		ret, e := Upload(fileUrl, baseName, fi.Reader, fi.IsGzipped, fi.MimeType, nil, jwt)
 		if e != nil {
 			return 0, e
 		}
@@ -142,26 +203,28 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 }
 
 func upload_one_chunk(filename string, reader io.Reader, master,
-	replication string, collection string, ttl string, jwt security.EncodedJwt,
-) (fid string, size uint32, e error) {
-	ret, err := Assign(master, 1, replication, collection, ttl)
-	if err != nil {
-		return "", 0, err
-	}
-	fileUrl, fid := "http://"+ret.Url+"/"+ret.Fid, ret.Fid
+	fileUrl string, jwt security.EncodedJwt,
+) (size uint32, e error) {
 	glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
 	uploadResult, uploadError := Upload(fileUrl, filename, reader, false,
-		"application/octet-stream", jwt)
+		"application/octet-stream", nil, jwt)
 	if uploadError != nil {
-		return fid, 0, uploadError
+		return 0, uploadError
 	}
-	return fid, uploadResult.Size, nil
+	return uploadResult.Size, nil
 }
 
-func upload_file_id_list(fileUrl, filename string, fids []string, jwt security.EncodedJwt) error {
-	var buf bytes.Buffer
-	buf.WriteString(strings.Join(fids, "\n"))
-	glog.V(4).Info("Uploading final list ", filename, " to ", fileUrl, "...")
-	_, e := Upload(fileUrl, filename, &buf, false, "text/plain", jwt)
+func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt security.EncodedJwt) error {
+	buf, e := manifest.Marshal()
+	if e != nil {
+		return e
+	}
+	bufReader := bytes.NewReader(buf)
+	glog.V(4).Info("Uploading chunks manifest ", manifest.Name, " to ", fileUrl, "...")
+	u, _ := url.Parse(fileUrl)
+	q := u.Query()
+	q.Set("cm", "true")
+	u.RawQuery = q.Encode()
+	_, e = Upload(u.String(), manifest.Name, bufReader, false, "application/json", nil, jwt)
 	return e
 }

weed/operation/sync_volume.go

@@ -0,0 +1,54 @@
+package operation
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+type SyncVolumeResponse struct {
+	Replication     string `json:"Replication,omitempty"`
+	Ttl             string `json:"Ttl,omitempty"`
+	TailOffset      uint64 `json:"TailOffset,omitempty"`
+	CompactRevision uint16 `json:"CompactRevision,omitempty"`
+	IdxFileSize     uint64 `json:"IdxFileSize,omitempty"`
+	Error           string `json:"error,omitempty"`
+}
+
+func GetVolumeSyncStatus(server string, vid string) (*SyncVolumeResponse, error) {
+	values := make(url.Values)
+	values.Add("volume", vid)
+	jsonBlob, err := util.Post("http://"+server+"/admin/sync/status", values)
+	glog.V(2).Info("sync volume result :", string(jsonBlob))
+	if err != nil {
+		return nil, err
+	}
+	var ret SyncVolumeResponse
+	err = json.Unmarshal(jsonBlob, &ret)
+	if err != nil {
+		return nil, err
+	}
+	if ret.Error != "" {
+		return nil, fmt.Errorf("Volume %s get sync status error: %s", vid, ret.Error)
+	}
+	return &ret, nil
+}
+
+func GetVolumeIdxEntries(server string, vid string, eachEntryFn func(key uint64, offset, size uint32)) error {
+	values := make(url.Values)
+	values.Add("volume", vid)
+	line := make([]byte, 16)
+	err := util.GetBufferStream("http://"+server+"/admin/sync/index", values, line, func(bytes []byte) {
+		key := util.BytesToUint64(bytes[:8])
+		offset := util.BytesToUint32(bytes[8:12])
+		size := util.BytesToUint32(bytes[12:16])
+		eachEntryFn(key, offset, size)
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}

weed/operation/upload_content.go

@@ -14,8 +14,8 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/chrislusf/seaweedfs/go/glog"
-	"github.com/chrislusf/seaweedfs/go/security"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/security"
 )
 
 type UploadResult struct {
@@ -36,13 +36,13 @@ func init() {
 
 var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
-func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, jwt security.EncodedJwt) (*UploadResult, error) {
+func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
 	return upload_content(uploadUrl, func(w io.Writer) (err error) {
 		_, err = io.Copy(w, reader)
 		return
-	}, filename, isGzipped, mtype, jwt)
+	}, filename, isGzipped, mtype, pairMap, jwt)
 }
-func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, jwt security.EncodedJwt) (*UploadResult, error) {
+func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
 	body_buf := bytes.NewBufferString("")
 	body_writer := multipart.NewWriter(body_buf)
 	h := make(textproto.MIMEHeader)
@@ -59,6 +59,7 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 	if jwt != "" {
 		h.Set("Authorization", "BEARER "+string(jwt))
 	}
+
 	file_writer, cp_err := body_writer.CreatePart(h)
 	if cp_err != nil {
 		glog.V(0).Infoln("error creating form file", cp_err.Error())
@@ -73,7 +74,17 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 		glog.V(0).Infoln("error closing body", err)
 		return nil, err
 	}
-	resp, post_err := client.Post(uploadUrl, content_type, body_buf)
+
+	req, postErr := http.NewRequest("POST", uploadUrl, body_buf)
+	if postErr != nil {
+		glog.V(0).Infoln("failing to upload to", uploadUrl, postErr.Error())
+		return nil, postErr
+	}
+	req.Header.Set("Content-Type", content_type)
+	for k, v := range pairMap {
+		req.Header.Set(k, v)
+	}
+	resp, post_err := client.Do(req)
 	if post_err != nil {
 		glog.V(0).Infoln("failing to upload to", uploadUrl, post_err.Error())
 		return nil, post_err
@@ -86,7 +97,7 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 	var ret UploadResult
 	unmarshal_err := json.Unmarshal(resp_body, &ret)
 	if unmarshal_err != nil {
-		glog.V(0).Infoln("failing to read upload resonse", uploadUrl, string(resp_body))
+		glog.V(0).Infoln("failing to read upload response", uploadUrl, string(resp_body))
 		return nil, unmarshal_err
 	}
 	if ret.Error != "" {

weed/pb/Makefile

@@ -0,0 +1,6 @@
+all: gen
+
+.PHONY : gen
+
+gen:
+	protoc seaweed.proto --go_out=plugins=grpc:.

weed/pb/seaweed.pb.go

@@ -0,0 +1,384 @@
+// Code generated by protoc-gen-go.
+// source: seaweed.proto
+// DO NOT EDIT!
+
+/*
+Package pb is a generated protocol buffer package.
+
+It is generated from these files:
+	seaweed.proto
+
+It has these top-level messages:
+	Heartbeat
+	HeartbeatResponse
+	VolumeInformationMessage
+*/
+package pb
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+import (
+	context "golang.org/x/net/context"
+	grpc "google.golang.org/grpc"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+type Heartbeat struct {
+	Ip             string                      `protobuf:"bytes,1,opt,name=ip" json:"ip,omitempty"`
+	Port           uint32                      `protobuf:"varint,2,opt,name=port" json:"port,omitempty"`
+	PublicUrl      string                      `protobuf:"bytes,3,opt,name=public_url,json=publicUrl" json:"public_url,omitempty"`
+	MaxVolumeCount uint32                      `protobuf:"varint,4,opt,name=max_volume_count,json=maxVolumeCount" json:"max_volume_count,omitempty"`
+	MaxFileKey     uint64                      `protobuf:"varint,5,opt,name=max_file_key,json=maxFileKey" json:"max_file_key,omitempty"`
+	DataCenter     string                      `protobuf:"bytes,6,opt,name=data_center,json=dataCenter" json:"data_center,omitempty"`
+	Rack           string                      `protobuf:"bytes,7,opt,name=rack" json:"rack,omitempty"`
+	AdminPort      uint32                      `protobuf:"varint,8,opt,name=admin_port,json=adminPort" json:"admin_port,omitempty"`
+	Volumes        []*VolumeInformationMessage `protobuf:"bytes,9,rep,name=volumes" json:"volumes,omitempty"`
+}
+
+func (m *Heartbeat) Reset()                    { *m = Heartbeat{} }
+func (m *Heartbeat) String() string            { return proto.CompactTextString(m) }
+func (*Heartbeat) ProtoMessage()               {}
+func (*Heartbeat) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+func (m *Heartbeat) GetIp() string {
+	if m != nil {
+		return m.Ip
+	}
+	return ""
+}
+
+func (m *Heartbeat) GetPort() uint32 {
+	if m != nil {
+		return m.Port
+	}
+	return 0
+}
+
+func (m *Heartbeat) GetPublicUrl() string {
+	if m != nil {
+		return m.PublicUrl
+	}
+	return ""
+}
+
+func (m *Heartbeat) GetMaxVolumeCount() uint32 {
+	if m != nil {
+		return m.MaxVolumeCount
+	}
+	return 0
+}
+
+func (m *Heartbeat) GetMaxFileKey() uint64 {
+	if m != nil {
+		return m.MaxFileKey
+	}
+	return 0
+}
+
+func (m *Heartbeat) GetDataCenter() string {
+	if m != nil {
+		return m.DataCenter
+	}
+	return ""
+}
+
+func (m *Heartbeat) GetRack() string {
+	if m != nil {
+		return m.Rack
+	}
+	return ""
+}
+
+func (m *Heartbeat) GetAdminPort() uint32 {
+	if m != nil {
+		return m.AdminPort
+	}
+	return 0
+}
+
+func (m *Heartbeat) GetVolumes() []*VolumeInformationMessage {
+	if m != nil {
+		return m.Volumes
+	}
+	return nil
+}
+
+type HeartbeatResponse struct {
+	VolumeSizeLimit uint64 `protobuf:"varint,1,opt,name=volumeSizeLimit" json:"volumeSizeLimit,omitempty"`
+	SecretKey       string `protobuf:"bytes,2,opt,name=secretKey" json:"secretKey,omitempty"`
+	Leader          string `protobuf:"bytes,3,opt,name=leader" json:"leader,omitempty"`
+}
+
+func (m *HeartbeatResponse) Reset()                    { *m = HeartbeatResponse{} }
+func (m *HeartbeatResponse) String() string            { return proto.CompactTextString(m) }
+func (*HeartbeatResponse) ProtoMessage()               {}
+func (*HeartbeatResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
+
+func (m *HeartbeatResponse) GetVolumeSizeLimit() uint64 {
+	if m != nil {
+		return m.VolumeSizeLimit
+	}
+	return 0
+}
+
+func (m *HeartbeatResponse) GetSecretKey() string {
+	if m != nil {
+		return m.SecretKey
+	}
+	return ""
+}
+
+func (m *HeartbeatResponse) GetLeader() string {
+	if m != nil {
+		return m.Leader
+	}
+	return ""
+}
+
+type VolumeInformationMessage struct {
+	Id               uint32 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"`
+	Size             uint64 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"`
+	Collection       string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"`
+	FileCount        uint64 `protobuf:"varint,4,opt,name=file_count,json=fileCount" json:"file_count,omitempty"`
+	DeleteCount      uint64 `protobuf:"varint,5,opt,name=delete_count,json=deleteCount" json:"delete_count,omitempty"`
+	DeletedByteCount uint64 `protobuf:"varint,6,opt,name=deleted_byte_count,json=deletedByteCount" json:"deleted_byte_count,omitempty"`
+	ReadOnly         bool   `protobuf:"varint,7,opt,name=read_only,json=readOnly" json:"read_only,omitempty"`
+	ReplicaPlacement uint32 `protobuf:"varint,8,opt,name=replica_placement,json=replicaPlacement" json:"replica_placement,omitempty"`
+	Version          uint32 `protobuf:"varint,9,opt,name=version" json:"version,omitempty"`
+	Ttl              uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"`
+}
+
+func (m *VolumeInformationMessage) Reset()                    { *m = VolumeInformationMessage{} }
+func (m *VolumeInformationMessage) String() string            { return proto.CompactTextString(m) }
+func (*VolumeInformationMessage) ProtoMessage()               {}
+func (*VolumeInformationMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
+
+func (m *VolumeInformationMessage) GetId() uint32 {
+	if m != nil {
+		return m.Id
+	}
+	return 0
+}
+
+func (m *VolumeInformationMessage) GetSize() uint64 {
+	if m != nil {
+		return m.Size
+	}
+	return 0
+}
+
+func (m *VolumeInformationMessage) GetCollection() string {
+	if m != nil {
+		return m.Collection
+	}
+	return ""
+}
+
+func (m *VolumeInformationMessage) GetFileCount() uint64 {
+	if m != nil {
+		return m.FileCount
+	}
+	return 0
+}
+
+func (m *VolumeInformationMessage) GetDeleteCount() uint64 {
+	if m != nil {
+		return m.DeleteCount
+	}
+	return 0
+}
+
+func (m *VolumeInformationMessage) GetDeletedByteCount() uint64 {
+	if m != nil {
+		return m.DeletedByteCount
+	}
+	return 0
+}
+
+func (m *VolumeInformationMessage) GetReadOnly() bool {
+	if m != nil {
+		return m.ReadOnly
+	}
+	return false
+}
+
+func (m *VolumeInformationMessage) GetReplicaPlacement() uint32 {
+	if m != nil {
+		return m.ReplicaPlacement
+	}
+	return 0
+}
+
+func (m *VolumeInformationMessage) GetVersion() uint32 {
+	if m != nil {
+		return m.Version
+	}
+	return 0
+}
+
+func (m *VolumeInformationMessage) GetTtl() uint32 {
+	if m != nil {
+		return m.Ttl
+	}
+	return 0
+}
+
+func init() {
+	proto.RegisterType((*Heartbeat)(nil), "pb.Heartbeat")
+	proto.RegisterType((*HeartbeatResponse)(nil), "pb.HeartbeatResponse")
+	proto.RegisterType((*VolumeInformationMessage)(nil), "pb.VolumeInformationMessage")
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// Client API for Seaweed service
+
+type SeaweedClient interface {
+	SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error)
+}
+
+type seaweedClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewSeaweedClient(cc *grpc.ClientConn) SeaweedClient {
+	return &seaweedClient{cc}
+}
+
+func (c *seaweedClient) SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error) {
+	stream, err := grpc.NewClientStream(ctx, &_Seaweed_serviceDesc.Streams[0], c.cc, "/pb.Seaweed/SendHeartbeat", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &seaweedSendHeartbeatClient{stream}
+	return x, nil
+}
+
+type Seaweed_SendHeartbeatClient interface {
+	Send(*Heartbeat) error
+	Recv() (*HeartbeatResponse, error)
+	grpc.ClientStream
+}
+
+type seaweedSendHeartbeatClient struct {
+	grpc.ClientStream
+}
+
+func (x *seaweedSendHeartbeatClient) Send(m *Heartbeat) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *seaweedSendHeartbeatClient) Recv() (*HeartbeatResponse, error) {
+	m := new(HeartbeatResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// Server API for Seaweed service
+
+type SeaweedServer interface {
+	SendHeartbeat(Seaweed_SendHeartbeatServer) error
+}
+
+func RegisterSeaweedServer(s *grpc.Server, srv SeaweedServer) {
+	s.RegisterService(&_Seaweed_serviceDesc, srv)
+}
+
+func _Seaweed_SendHeartbeat_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(SeaweedServer).SendHeartbeat(&seaweedSendHeartbeatServer{stream})
+}
+
+type Seaweed_SendHeartbeatServer interface {
+	Send(*HeartbeatResponse) error
+	Recv() (*Heartbeat, error)
+	grpc.ServerStream
+}
+
+type seaweedSendHeartbeatServer struct {
+	grpc.ServerStream
+}
+
+func (x *seaweedSendHeartbeatServer) Send(m *HeartbeatResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *seaweedSendHeartbeatServer) Recv() (*Heartbeat, error) {
+	m := new(Heartbeat)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _Seaweed_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "pb.Seaweed",
+	HandlerType: (*SeaweedServer)(nil),
+	Methods:     []grpc.MethodDesc{},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "SendHeartbeat",
+			Handler:       _Seaweed_SendHeartbeat_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "seaweed.proto",
+}
+
+func init() { proto.RegisterFile("seaweed.proto", fileDescriptor0) }
+
+var fileDescriptor0 = []byte{
+	// 498 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x74, 0x93, 0xdf, 0x6e, 0xd3, 0x30,
+	0x14, 0xc6, 0x49, 0x1a, 0xda, 0xe6, 0x74, 0x1d, 0x9d, 0x25, 0x90, 0x05, 0x03, 0x4a, 0xaf, 0x22,
+	0x81, 0x2a, 0x34, 0x24, 0x6e, 0xb8, 0x63, 0xd2, 0x04, 0x1a, 0x88, 0xc9, 0x15, 0xdc, 0x46, 0x4e,
+	0x72, 0x86, 0xac, 0x39, 0x7f, 0xe4, 0xb8, 0xa3, 0xd9, 0x03, 0xf1, 0x24, 0x3c, 0x18, 0xf2, 0x71,
+	0xd2, 0x02, 0x12, 0x77, 0xe7, 0xfc, 0xce, 0x17, 0xfb, 0xf4, 0xfb, 0x5c, 0x98, 0xb7, 0x28, 0x7f,
+	0x20, 0x16, 0xeb, 0xc6, 0xd4, 0xb6, 0x66, 0x61, 0x93, 0xad, 0x7e, 0x86, 0x10, 0x7f, 0x40, 0x69,
+	0x6c, 0x86, 0xd2, 0xb2, 0x63, 0x08, 0x55, 0xc3, 0x83, 0x65, 0x90, 0xc4, 0x22, 0x54, 0x0d, 0x63,
+	0x10, 0x35, 0xb5, 0xb1, 0x3c, 0x5c, 0x06, 0xc9, 0x5c, 0x50, 0xcd, 0x9e, 0x02, 0x34, 0xdb, 0x4c,
+	0xab, 0x3c, 0xdd, 0x1a, 0xcd, 0x47, 0xa4, 0x8d, 0x3d, 0xf9, 0x6a, 0x34, 0x4b, 0x60, 0x51, 0xca,
+	0x5d, 0x7a, 0x5b, 0xeb, 0x6d, 0x89, 0x69, 0x5e, 0x6f, 0x2b, 0xcb, 0x23, 0xfa, 0xfc, 0xb8, 0x94,
+	0xbb, 0x6f, 0x84, 0xcf, 0x1d, 0x65, 0x4b, 0x38, 0x72, 0xca, 0x6b, 0xa5, 0x31, 0xbd, 0xc1, 0x8e,
+	0xdf, 0x5f, 0x06, 0x49, 0x24, 0xa0, 0x94, 0xbb, 0x0b, 0xa5, 0xf1, 0x12, 0x3b, 0xf6, 0x1c, 0x66,
+	0x85, 0xb4, 0x32, 0xcd, 0xb1, 0xb2, 0x68, 0xf8, 0x98, 0xee, 0x02, 0x87, 0xce, 0x89, 0xb8, 0xfd,
+	0x8c, 0xcc, 0x6f, 0xf8, 0x84, 0x26, 0x54, 0xbb, 0xfd, 0x64, 0x51, 0xaa, 0x2a, 0xa5, 0xcd, 0xa7,
+	0x74, 0x75, 0x4c, 0xe4, 0xca, 0xad, 0xff, 0x16, 0x26, 0x7e, 0xb7, 0x96, 0xc7, 0xcb, 0x51, 0x32,
+	0x3b, 0x3b, 0x5d, 0x37, 0xd9, 0xda, 0xef, 0xf5, 0xb1, 0xba, 0xae, 0x4d, 0x29, 0xad, 0xaa, 0xab,
+	0xcf, 0xd8, 0xb6, 0xf2, 0x3b, 0x8a, 0x41, 0xbc, 0x6a, 0xe1, 0x64, 0xef, 0x93, 0xc0, 0xb6, 0xa9,
+	0xab, 0x16, 0x59, 0x02, 0x0f, 0xfc, 0x7c, 0xa3, 0xee, 0xf0, 0x93, 0x2a, 0x95, 0x25, 0xf3, 0x22,
+	0xf1, 0x2f, 0x66, 0xa7, 0x10, 0xb7, 0x98, 0x1b, 0xb4, 0x97, 0xd8, 0x91, 0x9d, 0xb1, 0x38, 0x00,
+	0xf6, 0x08, 0xc6, 0x1a, 0x65, 0x81, 0xa6, 0xf7, 0xb3, 0xef, 0x56, 0xbf, 0x42, 0xe0, 0xff, 0x5b,
+	0x8d, 0xc2, 0x2a, 0xe8, 0xbe, 0xb9, 0x08, 0x55, 0xe1, 0xcc, 0x68, 0xd5, 0x1d, 0xd2, 0xe9, 0x91,
+	0xa0, 0x9a, 0x3d, 0x03, 0xc8, 0x6b, 0xad, 0x31, 0x77, 0x1f, 0xf6, 0x87, 0xff, 0x41, 0x9c, 0x59,
+	0xe4, 0xff, 0x21, 0xa7, 0x48, 0xc4, 0x8e, 0xf8, 0x88, 0x5e, 0xc0, 0x51, 0x81, 0x1a, 0xed, 0x20,
+	0xf0, 0x11, 0xcd, 0x3c, 0xf3, 0x92, 0x57, 0xc0, 0x7c, 0x5b, 0xa4, 0x59, 0xb7, 0x17, 0x8e, 0x49,
+	0xb8, 0xe8, 0x27, 0xef, 0xbb, 0x41, 0xfd, 0x04, 0x62, 0x83, 0xb2, 0x48, 0xeb, 0x4a, 0x77, 0x94,
+	0xda, 0x54, 0x4c, 0x1d, 0xf8, 0x52, 0xe9, 0x8e, 0xbd, 0x84, 0x13, 0x83, 0x8d, 0x56, 0xb9, 0x4c,
+	0x1b, 0x2d, 0x73, 0x2c, 0xb1, 0x1a, 0x02, 0x5c, 0xf4, 0x83, 0xab, 0x81, 0x33, 0x0e, 0x93, 0x5b,
+	0x34, 0xad, 0xfb, 0x59, 0x31, 0x49, 0x86, 0x96, 0x2d, 0x60, 0x64, 0xad, 0xe6, 0x40, 0xd4, 0x95,
+	0x67, 0x17, 0x30, 0xd9, 0xf8, 0x97, 0xcf, 0xde, 0xc1, 0x7c, 0x83, 0x55, 0x71, 0x78, 0xf2, 0x73,
+	0x17, 0xff, 0xbe, 0x7d, 0xfc, 0xf0, 0xaf, 0x76, 0x08, 0x7a, 0x75, 0x2f, 0x09, 0x5e, 0x07, 0xd9,
+	0x98, 0xfe, 0x37, 0x6f, 0x7e, 0x07, 0x00, 0x00, 0xff, 0xff, 0x7c, 0xda, 0x3f, 0x9c, 0x48, 0x03,
+	0x00, 0x00,
+}