1.66

fix https://github.com/chrislusf/seaweedfs/issues/1241

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,33 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Sponsors SeaweedFS via Patreon https://www.patreon.com/seaweedfs
+
+example of a good issue report:
+https://github.com/chrislusf/seaweedfs/issues/1005
+example of a bad issue report:
+https://github.com/chrislusf/seaweedfs/issues/1008
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**System Setup**
+- List the command line to start "weed master", "weed volume", "weed filer", "weed s3", "weed mount".
+- OS version
+- output of `weed version`
+- if using filer, show the content of `filer.toml`
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

.gitignore

@@ -1,3 +1,85 @@
-go/weed/.goxc*
+.goxc*
+vendor
 tags
 *.swp
+### OSX template
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio
+
+*.iml
+
+## Directory-based project format:
+.idea/
+# if you remove the above rule, at least ignore the following:
+
+# User-specific stuff:
+# .idea/workspace.xml
+# .idea/tasks.xml
+# .idea/dictionaries
+
+# Sensitive or high-churn files:
+# .idea/dataSources.ids
+# .idea/dataSources.xml
+# .idea/sqlDataSources.xml
+# .idea/dynamic.xml
+# .idea/uiDesigner.xml
+
+# Gradle:
+# .idea/gradle.xml
+# .idea/libraries
+
+# Mongo Explorer plugin:
+# .idea/mongoSettings.xml
+
+## File-based project format:
+*.ipr
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+
+test_data
+build
+target
+*.class
+other/java/hdfs/dependency-reduced-pom.xml
+
+# binary file
+weed/weed

.project

@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
-	<name>weed-fs</name>
-	<comment></comment>
-	<projects>
-	</projects>
-	<buildSpec>
-	</buildSpec>
-	<natures>
-		<nature>goclipse.goNature</nature>
-		<nature>org.eclipse.wst.common.project.facet.core.nature</nature>
-	</natures>
-</projectDescription>

.travis.yml

@@ -1,14 +1,48 @@
+sudo: false
 language: go
-go: 
- - 1.3
- - 1.4.1
- - tip
+go:
+  - 1.12.x
+  - 1.13.x
+  - 1.14.x
 
 before_install:
- - export PATH=/home/travis/gopath/bin:$PATH
+  - export PATH=/home/travis/gopath/bin:$PATH
 
 install:
- - go get ./go/...
- 
+  - export CGO_ENABLED="0"
+  - go env
+
 script:
- - go test ./go/...
+  - env GO111MODULE=on go test ./weed/...
+
+before_deploy:
+  - make release
+deploy:
+  provider: releases
+  skip_cleanup: true
+  api_key:
+    secure: ERL986+ncQ8lwAJUYDrQ8s2/FxF/cyNIwJIFCqspnWxQgGNNyokET9HapmlPSxjpFRF0q6L2WCg9OY3mSVRq4oI6hg1igOQ12KlLyN71XSJ3c8w0Ay5ho48TQ9l3f3Iu97mntBCe9l0R9pnT8wj1VI8YJxloXwUMG2yeTjA9aBI=
+  file:
+    - build/linux_arm.tar.gz
+    - build/linux_arm64.tar.gz
+    - build/linux_386.tar.gz
+    - build/linux_amd64.tar.gz
+    - build/linux_amd64_large_disk.tar.gz
+    - build/darwin_amd64.tar.gz
+    - build/darwin_amd64_large_disk.tar.gz
+    - build/windows_386.zip
+    - build/windows_amd64.zip
+    - build/windows_amd64_large_disk.zip
+    - build/freebsd_arm.tar.gz
+    - build/freebsd_amd64.tar.gz
+    - build/freebsd_386.tar.gz
+    - build/netbsd_arm.tar.gz
+    - build/netbsd_amd64.tar.gz
+    - build/netbsd_386.tar.gz
+    - build/openbsd_arm.tar.gz
+    - build/openbsd_amd64.tar.gz
+    - build/openbsd_386.tar.gz
+  on:
+    tags: true
+    repo: chrislusf/seaweedfs
+    go: 1.14.x

Dockerfile

@@ -1,21 +0,0 @@
-FROM progrium/busybox
-
-WORKDIR /opt/weed
-
-RUN opkg-install curl
-RUN echo insecure >> ~/.curlrc
-
-RUN \
-  curl -Lks https://bintray.com$(curl -Lk http://bintray.com/chrislusf/Weed-FS/seaweed/_latestVersion | grep linux_amd64.tar.gz | sed -n "/href/ s/.*href=['\"]\([^'\"]*\)['\"].*/\1/gp") | gunzip | tar -xf - -C /opt/weed/ && \
-    mv weed_*/* /bin && \
-  chmod +x ./bin/weed
-
-EXPOSE 8080
-EXPOSE 9333
-
-VOLUME /data
-
-ENV WEED_HOME /opt/weed
-ENV PATH ${PATH}:${WEED_HOME}/bin
-
-ENTRYPOINT ["weed"]

Dockerfile.go_build

@@ -1,6 +0,0 @@
-FROM cydev/go
-RUN go get github.com/chrislusf/weed-fs/go/weed
-EXPOSE 8080
-EXPOSE 9333
-VOLUME /data
-ENTRYPOINT ["weed"]

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2016 Chris Lu
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -0,0 +1,130 @@
+BINARY = weed/weed
+package = github.com/chrislusf/seaweedfs/weed
+
+GO_FLAGS = #-v
+SOURCE_DIR = ./weed/
+
+appname := weed
+
+sources := $(wildcard *.go)
+
+build = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -ldflags "-extldflags -static" -o build/$(appname)$(3) $(SOURCE_DIR)
+tar = cd build && tar -cvzf $(1)_$(2).tar.gz $(appname)$(3) && rm $(appname)$(3)
+zip = cd build && zip $(1)_$(2).zip $(appname)$(3) && rm $(appname)$(3)
+
+build_large = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -tags 5BytesOffset -ldflags "-extldflags -static" -o build/$(appname)$(3) $(SOURCE_DIR)
+tar_large = cd build && tar -cvzf $(1)_$(2)_large_disk.tar.gz $(appname)$(3) && rm $(appname)$(3)
+zip_large = cd build && zip $(1)_$(2)_large_disk.zip $(appname)$(3) && rm $(appname)$(3)
+
+all: build
+
+.PHONY : clean deps build linux release windows_build darwin_build linux_build bsd_build clean
+
+clean:
+	go clean -i $(GO_FLAGS) $(SOURCE_DIR)
+	rm -f $(BINARY)
+	rm -rf build/
+
+deps:
+	go get $(GO_FLAGS) -d $(SOURCE_DIR)
+	rm -rf /home/travis/gopath/src/github.com/coreos/etcd/vendor/golang.org/x/net/trace
+	rm -rf /home/travis/gopath/src/go.etcd.io/etcd/vendor/golang.org/x/net/trace
+
+build: deps
+	go build $(GO_FLAGS) -o $(BINARY) $(SOURCE_DIR)
+
+linux: deps
+	mkdir -p linux
+	GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -o linux/$(BINARY) $(SOURCE_DIR)
+
+release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_darwin_build 5_byte_windows_build
+
+##### LINUX BUILDS #####
+5_byte_linux_build:
+	$(call build_large,linux,amd64,)
+	$(call tar_large,linux,amd64)
+
+5_byte_darwin_build:
+	$(call build_large,darwin,amd64,)
+	$(call tar_large,darwin,amd64)
+
+5_byte_windows_build:
+	$(call build_large,windows,amd64,.exe)
+	$(call zip_large,windows,amd64,.exe)
+
+linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz
+
+build/linux_386.tar.gz: $(sources)
+	$(call build,linux,386,)
+	$(call tar,linux,386)
+
+build/linux_amd64.tar.gz: $(sources)
+	$(call build,linux,amd64,)
+	$(call tar,linux,amd64)
+
+build/linux_arm.tar.gz: $(sources)
+	$(call build,linux,arm,)
+	$(call tar,linux,arm)
+
+build/linux_arm64.tar.gz: $(sources)
+	$(call build,linux,arm64,)
+	$(call tar,linux,arm64)
+
+##### DARWIN (MAC) BUILDS #####
+darwin_build: build/darwin_amd64.tar.gz
+
+build/darwin_amd64.tar.gz: $(sources)
+	$(call build,darwin,amd64,)
+	$(call tar,darwin,amd64)
+
+##### WINDOWS BUILDS #####
+windows_build: build/windows_386.zip build/windows_amd64.zip
+
+build/windows_386.zip: $(sources)
+	$(call build,windows,386,.exe)
+	$(call zip,windows,386,.exe)
+
+build/windows_amd64.zip: $(sources)
+	$(call build,windows,amd64,.exe)
+	$(call zip,windows,amd64,.exe)
+
+##### BSD BUILDS #####
+bsd_build: build/freebsd_arm.tar.gz build/freebsd_386.tar.gz build/freebsd_amd64.tar.gz \
+ build/netbsd_arm.tar.gz build/netbsd_386.tar.gz build/netbsd_amd64.tar.gz \
+ build/openbsd_arm.tar.gz build/openbsd_386.tar.gz build/openbsd_amd64.tar.gz
+
+build/freebsd_386.tar.gz: $(sources)
+	$(call build,freebsd,386,)
+	$(call tar,freebsd,386)
+
+build/freebsd_amd64.tar.gz: $(sources)
+	$(call build,freebsd,amd64,)
+	$(call tar,freebsd,amd64)
+
+build/freebsd_arm.tar.gz: $(sources)
+	$(call build,freebsd,arm,)
+	$(call tar,freebsd,arm)
+
+build/netbsd_386.tar.gz: $(sources)
+	$(call build,netbsd,386,)
+	$(call tar,netbsd,386)
+
+build/netbsd_amd64.tar.gz: $(sources)
+	$(call build,netbsd,amd64,)
+	$(call tar,netbsd,amd64)
+
+build/netbsd_arm.tar.gz: $(sources)
+	$(call build,netbsd,arm,)
+	$(call tar,netbsd,arm)
+
+build/openbsd_386.tar.gz: $(sources)
+	$(call build,openbsd,386,)
+	$(call tar,openbsd,386)
+
+build/openbsd_amd64.tar.gz: $(sources)
+	$(call build,openbsd,amd64,)
+	$(call tar,openbsd,amd64)
+
+build/openbsd_arm.tar.gz: $(sources)
+	$(call build,openbsd,arm,)
+	$(call tar,openbsd,arm)

README.md

@@ -1,58 +1,151 @@
-Seaweed File System
-=======
+# SeaweedFS
 
-[![Build Status](https://travis-ci.org/chrislusf/weed-fs.svg?branch=master)](https://travis-ci.org/chrislusf/weed-fs)
-[![GoDoc](https://godoc.org/github.com/chrislusf/weed-fs/go?status.svg)](https://godoc.org/github.com/chrislusf/weed-fs/go)
-[![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/weed-fs/wiki)
 
-![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/weed-fs/master/note/seaweedfs.png)
+[![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs)
+[![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
+[![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
+[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=604800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
 
-## Introduction
+![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)
+
+<h2 align="center">Supporting SeaweedFS</h2>
+
+SeaweedFS is an independent Apache-licensed open source project with its ongoing development made
+possible entirely thanks to the support of these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md).
+If you'd like to grow SeaweedFS even stronger, please consider joining our
+<a href="https://www.patreon.com/seaweedfs">sponsors on Patreon</a>.
+
+Your support will be really appreciated by me and other supporters!
+
+<h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h3>
+
+<!--
+<h4 align="center">Platinum</h4>
+
+<p align="center">
+  <a href="" target="_blank">
+    Add your name or icon here
+  </a>
+</p>
+
+<h4 align="center">Gold</h4>
+
+<table>
+  <tbody>
+    <tr>
+      <td align="center" valign="middle">
+        <a href="" target="_blank">
+          Add your name or icon here
+        </a>
+      </td>
+    </tr>
+    <tr></tr>
+  </tbody>
+</table>
+
+-->
+
+---
+
+
+- [Download Binaries for different platforms](https://github.com/chrislusf/seaweedfs/releases/latest)
+- [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
+- [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
+- [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
+- [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
+- [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
+
+Table of Contents
+=================
+
+* [Introduction](#introduction)
+* [Features](#features)
+    * [Additional Features](#additional-features)
+    * [Filer Features](#filer-features)
+* [Example Usage](#example-usage)
+* [Architecture](#architecture)
+* [Compared to Other File Systems](#compared-to-other-file-systems)
+    * [Compared to HDFS](#compared-to-hdfs)
+    * [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
+    * [Compared to GlusterFS](#compared-to-glusterfs)
+    * [Compared to Ceph](#compared-to-ceph)
+* [Dev Plan](#dev-plan)
+* [Installation Guide](#installation-guide)
+* [Disk Related Topics](#disk-related-topics)
+* [Benchmark](#Benchmark)
+* [License](#license)
+
+## Introduction ##
 
 SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
 
 1. to store billions of files!
 2. to serve the files fast!
 
-Instead of supporting full POSIX file system semantics, SeaweedFS choose to implement only a key~file mapping. Similar to the word "NoSQL", you can call it as "NoFS".
+SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages file volumes, and it lets these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (O(1), usually just one disk read operation).
 
-Instead of managing all file metadata in a central master, SeaweedFS choose to manages file volumes in the central master, and let volume servers manage files and the metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers' memories, allowing faster file access with just one disk read operation!
+SeaweedFS can transparently integrate with the cloud. With hot data on local cluster, and warm data on the cloud with O(1) access time, SeaweedFS can achieve both fast local access time and elastic cloud storage capacity, without any client side changes.
 
-SeaweedFS models after [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf).
+There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
 
-SeaweedFS costs only 40 bytes disk storage for each file's metadata. It is so simple with O(1) disk read that you are welcome to challenge the performance with your actual use cases.
+SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). Also, SeaweedFS implements erasure coding with ideas from [f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf)
 
+On top of the object store, optional [Filer] can support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql, Postgres, Redis, Etcd, Cassandra, LevelDB, MemSql, TiDB, TiKV, CockroachDB, etc.
 
-![](https://api.bintray.com/packages/chrislusf/Weed-FS/seaweed/images/download.png)
-https://bintray.com/chrislusf/Weed-FS/seaweed Download latest compiled binaries for different platforms here.
+[Back to TOC](#table-of-contents)
 
-http://groups.google.com/group/weed-file-system Seaweed File System Discussion Group
+## Additional Features ##
+* Can choose no replication or different replication levels, rack and data center aware.
+* Automatic master servers failover - no single point of failure (SPOF).
+* Automatic Gzip compression depending on file mime type.
+* Automatic compaction to reclaim disk space after deletion or update.
+* [Automatic entry TTL expiration][VolumeServerTTL].
+* Servers in the same cluster can have different disk spaces, file systems, OS etc.
+* Adding/Removing servers does **not** cause any data re-balancing.
+* Optionally fix the orientation for jpeg pictures.
+* Support ETag, Accept-Range, Last-Modified, etc.
+* Support in-memory/leveldb/readonly mode tuning for memory/performance balance.
+* Support rebalancing the writable and readonly volumes.
+* [Transparent cloud integration][CloudTier]: unlimited capacity via tiered cloud storage for warm data.
+* [Erasure Coding for warm storage][ErasureCoding]  Rack-Aware 10.4 erasure coding reduces storage cost and increases availability.
 
-## Additional Features
-* Can choose no replication or different replication level, rack and data center aware
-* Automatic master servers failover. No single point of failure, SPOF.
-* Automatic Gzip compression depending on file mime type
-* Automatic compaction to reclaimed disk spaces after deletion or update
-* Servers in the same cluster can have different disk spaces, different file systems, different OS
-* Adding/Removing servers do not cause any data re-balancing
-* Optional [filer server][Filer] provides "normal" directories and files via http
-* For jpeg pictures, optionally fix the orientation.
-* Support Etag, Accept-Range, Last-Modified, etc.
-* Support in-memory/leveldb/boltdb mode tuning for memory/performance balance.
+[Back to TOC](#table-of-contents)
 
-[Filer]: https://github.com/chrislusf/weed-fs/wiki/Filer
+## Filer Features ##
+* [filer server][Filer] provide "normal" directories and files via http.
+* [mount filer][Mount] to read and write files directly as a local directory via FUSE.
+* [Amazon S3 compatible API][AmazonS3API] to access files with S3 tooling.
+* [Hadoop Compatible File System][Hadoop] to access files from Hadoop/Spark/Flink/etc jobs.
+* [Async Backup To Cloud][BackupToCloud] has extremely fast local access and backups to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
+* [WebDAV] access as a mapped drive on Mac and Windows, or from mobile devices.
+* [AES256-GCM Encrypted Storage][FilerDataEncryption] safely stores the encrypted data.
+* [File TTL][FilerTTL] automatically purge file metadata and actual file data.
 
-## Example Usage
-By default, the master node runs on port 9333, and the volume nodes runs on port 8080.
-Here I will start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. Here I just use localhost as example.
+[Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files
+[Mount]: https://github.com/chrislusf/seaweedfs/wiki/Mount
+[AmazonS3API]: https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API
+[BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Backup-to-Cloud
+[Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
+[WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV
+[ErasureCoding]: https://github.com/chrislusf/seaweedfs/wiki/Erasure-coding-for-warm-storage
+[CloudTier]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Tier
+[FilerDataEncryption]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Data-Encryption
+[FilerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Stores
+[VolumeServerTTL]: https://github.com/chrislusf/seaweedfs/wiki/Store-file-with-a-Time-To-Live
 
-SeaweedFS uses HTTP REST operations to write, read, delete. The return results are JSON or JSONP format.
+[Back to TOC](#table-of-contents)
 
-### Start Master Server
+## Example Usage ##
+
+By default, the master node runs on port 9333, and the volume nodes run on port 8080.
+Let's start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. We'll use localhost as an example.
+
+SeaweedFS uses HTTP REST operations to read, write, and delete. The responses are in JSON or JSONP format.
+
+### Start Master Server ###
 
 ```
 > ./weed master
-
 ```
 
 ### Start Volume Servers ###
@@ -60,57 +153,58 @@ SeaweedFS uses HTTP REST operations to write, read, delete. The return results a
 ```
 > weed volume -dir="/tmp/data1" -max=5  -mserver="localhost:9333" -port=8080 &
 > weed volume -dir="/tmp/data2" -max=10 -mserver="localhost:9333" -port=8081 &
-
 ```
 
 ### Write File ###
 
-To upload a file, first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an fid and a volume server url:
+To upload a file: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server url:
 
 ```
-> curl -X POST http://localhost:9333/dir/assign
+> curl http://localhost:9333/dir/assign
 {"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
 ```
 
-Second, to store the file content, send a HTTP multipart PUT or POST request to `url + '/' + fid` from the response:
+Second, to store the file content, send a HTTP multi-part POST request to `url + '/' + fid` from the response:
 
 ```
-> curl -X PUT -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6
-{"size": 43234}
+> curl -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6
+{"name":"myphoto.jpg","size":43234,"eTag":"1cc0118e"}
 ```
 
-For update, send another PUT or POST request with updated file content.
+To update, send another POST request with updated file content.
 
 For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
 
 ```
 > curl -X DELETE http://127.0.0.1:8080/3,01637037d6
 ```
+
 ### Save File Id ###
-Now you can save the fid, 3,01637037d6 in this case, to some database field.
 
-The number 3 here, is a volume id. After the comma, it's one file key, 01, and a file cookie, 637037d6.
+Now, you can save the `fid`, 3,01637037d6 in this case, to a database field.
 
-The volume id is an unsigned 32 bit integer. The file key is an unsigned 64bit integer. The file cookie is an unsigned 32bit integer, used to prevent URL guessing.
+The number 3 at the start represents a volume id. After the comma, it's one file key, 01, and a file cookie, 637037d6.
 
-The file key and file cookie are both coded in hex. You can store the <volume id, file key, file cookie> tuple in your own format, or simply store the fid as string.
+The volume id is an unsigned 32-bit integer. The file key is an unsigned 64-bit integer. The file cookie is an unsigned 32-bit integer, used to prevent URL guessing.
 
-If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most usage would not need 2^32 volumes.
+The file key and file cookie are both coded in hex. You can store the <volume id, file key, file cookie> tuple in your own format, or simply store the `fid` as a string.
 
-If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, 4-byte integer for file cookie. So 16 bytes are enough (more than enough).
+If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most uses will not need 2^32 volumes.
+
+If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, and a 4-byte integer for the file cookie. So 16 bytes are more than enough.
 
 ### Read File ###
 
-Here is the example on how to render the URL.
+Here is an example of how to render the URL.
 
-First lookup the volume server's URLs by the file's volumeId:
+First look up the volume server's URLs by the file's volumeId:
 
 ```
 > curl http://localhost:9333/dir/lookup?volumeId=3
-{"locations":[{"publicUrl":"localhost:8080","url":"localhost:8080"}]}
+{"volumeId":"3","locations":[{"publicUrl":"localhost:8080","url":"localhost:8080"}]}
 ```
 
-(However, since usually there are not too many volume servers, and volumes does not move often, you can cache the results most of the time. Depends on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read.)
+Since (usually) there are not too many volume servers, and volumes don't move often, you can cache the results most of the time. Depending on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read.
 
 Now you can take the public url, render the url or directly read from the volume server via url:
 
@@ -118,7 +212,7 @@ Now you can take the public url, render the url or directly read from the volume
  http://localhost:8080/3,01637037d6.jpg
 ```
 
-Notice we add an file extension ".jpg" here. It's optional and just one way for the client to specify the file content type.
+Notice we add a file extension ".jpg" here. It's optional and just one way for the client to specify the file content type.
 
 If you want a nicer URL, you can use one of these alternative URL formats:
 
@@ -130,14 +224,23 @@ If you want a nicer URL, you can use one of these alternative URL formats:
  http://localhost:8080/3,01637037d6
 ```
 
+If you want to get a scaled version of an image, you can add some params:
+
+```
+http://localhost:8080/3/01637037d6.jpg?height=200&width=200
+http://localhost:8080/3/01637037d6.jpg?height=200&width=200&mode=fit
+http://localhost:8080/3/01637037d6.jpg?height=200&width=200&mode=fill
+```
+
 ### Rack-Aware and Data Center-Aware Replication ###
-SeaweedFS apply the replication strategy on a volume level. So when you are getting a file id, you can specify the replication strategy. For example:
+
+SeaweedFS applies the replication strategy at a volume level. So, when you are getting a file id, you can specify the replication strategy. For example:
 
 ```
-curl -X POST http://localhost:9333/dir/assign?replication=001
+curl http://localhost:9333/dir/assign?replication=001
 ```
 
-Here is the meaning of the replication parameter:
+The replication parameter options are:
 
 ```
 000: no replication
@@ -150,22 +253,20 @@ Here is the meaning of the replication parameter:
 
 More details about replication can be found [on the wiki][Replication].
 
-[Replication]: https://github.com/chrislusf/weed-fs/wiki/Replication
+[Replication]: https://github.com/chrislusf/seaweedfs/wiki/Replication
 
 You can also set the default replication strategy when starting the master server.
 
-### Allocate File Key on specific data center ###
+### Allocate File Key on Specific Data Center ###
 
-Volume servers can start with a specific data center name:
+Volume servers can be started with a specific data center name:
 
 ```
  weed volume -dir=/tmp/1 -port=8080 -dataCenter=dc1
  weed volume -dir=/tmp/2 -port=8081 -dataCenter=dc2
 ```
 
-Or the master server can determine the data center via volume server's IP address and settings in weed.conf file.
-
-Now when requesting a file key, an optional "dataCenter" parameter can limit the assigned volume to the specific data center. For example, this specifies that the assigned volume should be limited to 'dc1':
+When requesting a file key, an optional "dataCenter" parameter can limit the assigned volume to the specific data center. For example, this specifies that the assigned volume should be limited to 'dc1':
 
 ```
  http://localhost:9333/dir/assign?dataCenter=dc1
@@ -177,193 +278,244 @@ Now when requesting a file key, an optional "dataCenter" parameter can limit the
   * [Chunking large files][feat-3]
   * [Collection as a Simple Name Space][feat-4]
 
-[feat-1]: https://github.com/chrislusf/weed-fs/wiki/Failover-Master-Server
-[feat-2]: https://github.com/chrislusf/weed-fs/wiki/Optimization#insert-with-your-own-keys
-[feat-3]: https://github.com/chrislusf/weed-fs/wiki/Optimization#upload-large-files
-[feat-4]: https://github.com/chrislusf/weed-fs/wiki/Optimization#collection-as-a-simple-name-space
+[feat-1]: https://github.com/chrislusf/seaweedfs/wiki/Failover-Master-Server
+[feat-2]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#insert-with-your-own-keys
+[feat-3]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#upload-large-files
+[feat-4]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#collection-as-a-simple-name-space
+
+[Back to TOC](#table-of-contents)
 
 ## Architecture ##
-Usually distributed file system split each file into chunks, and a central master keeps a mapping of a filename and a chunk index to chunk handles, and also which chunks each chunk server has.
 
-This has the draw back that the central master can not handle many small files efficiently, and since all read requests need to go through the chunk master, responses would be slow for many concurrent web users.
+Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.
 
-Instead of managing chunks, SeaweedFS choose to manage data volumes in the master server. Each data volume is size 32GB, and can hold a lot of files. And each storage node can has many data volumes. So the master node only needs to store the metadata about the volumes, which is fairly small amount of data and pretty static most of the time.
+The main drawback is that the central master can't handle many small files efficiently, and since all read requests need to go through the chunk master, so it might not scale well for many concurrent users.
 
-The actual file metadata is stored in each volume on volume servers. Since each volume server only manage metadata of files on its own disk, and only 16 bytes for each file, all file access can read file metadata just from memory and only needs one disk operation to actually read file data.
+Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is 32GB in size, and can hold a lot of files. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is a fairly small amount of data and is generally stable.
+
+The actual file metadata is stored in each volume on volume servers. Since each volume server only manages metadata of files on its own disk, with only 16 bytes for each file, all file access can read file metadata just from memory and only needs one disk operation to actually read file data.
 
 For comparison, consider that an xfs inode structure in Linux is 536 bytes.
 
 ### Master Server and Volume Server ###
+
 The architecture is fairly simple. The actual data is stored in volumes on storage nodes. One volume server can have multiple volumes, and can both support read and write access with basic authentication.
 
-All volumes are managed by a master server. The master server contains volume id to volume server mapping. This is fairly static information, and could be cached easily.
-
-On each write request, the master server also generates a file key, which is a growing 64bit unsigned integer. Since the write requests are not as busy as read requests, one master server should be able to handle the concurrency well.
+All volumes are managed by a master server. The master server contains the volume id to volume server mapping. This is fairly static information, and can be easily cached.
 
+On each write request, the master server also generates a file key, which is a growing 64-bit unsigned integer. Since write requests are not generally as frequent as read requests, one master server should be able to handle the concurrency well.
 
 ### Write and Read files ###
 
-When a client sends a write request, the master server returns <volume id, file key, file cookie, volume node url> for the file. The client then contact the volume node and POST the file content via REST.
+When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node url) for the file. The client then contacts the volume node and POSTs the file content.
 
-When a client needs to read a file based on <volume id, file key, file cookie>, it can ask the master server by the <volum id> for the <volume node url, volume node public url>, or from cache. Then the client can HTTP GET the content via REST, or just render the URL on web pages and let browsers to fetch the content.
+When a client needs to read a file based on (volume id, file key, file cookie), it asks the master server by the volume id for the (volume node url, volume node public url), or retrieves this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
 
-Please see the example for details on write-read process.
+Please see the example for details on the write-read process.
 
 ### Storage Size ###
-In current implementation, each volume can be size of 8x2^32 bytes (32GiB). This is because of aligning contents to 8 bytes. We can be easily increased to 64G, or 128G, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
 
-There can be 2^32 volumes. So total system size is 8 x 2^32 bytes x 2^32 = 8 x 4GiB x 4Gi = 128EiB (2^67 bytes, or 128 exbibytes).
+In the current implementation, each volume can hold 32 gibibytes (32GiB or 8x2^32 bytes). This is because we align content to 8 bytes. We can easily increase this to 64GiB, or 128GiB, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
+
+There can be 4 gibibytes (4GiB or 2^32 bytes) of volumes. So the total system size is 8 x 4GiB x 4GiB which is 128 exbibytes (128EiB or 2^67 bytes).
 
 Each individual file size is limited to the volume size.
 
 ### Saving memory ###
-All file meta information on volume server is readable from memory without disk access. Each file just takes an 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own the space cost for the map. But usually the disk runs out before the memory does.
 
+All file meta information stored on an volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
 
-## Compared to Other File Systems##
-Frankly, I don't use other distributed file systems too often. All seems more complicated than necessary. Please correct me if anything here is wrong.
+### Tiered Storage to the cloud ###
 
-### Compared to Ceph ###
-Ceph can be setup similar to SeaweedFS as a key~blob store. It is much more complicated, with the need to support layers on top of it. Here is a more detailed comparison. https://code.google.com/p/weed-fs/issues/detail?id=44
+The local volume servers are much faster, while cloud storages have elastic capacity and are actually more cost-efficient if not accessed often (usually free to upload, but relatively costly to access). With the append-only structure and O(1) access time, SeaweedFS can take advantage of both local and cloud storage by offloading the warm data to the cloud.
 
-SeaweedFS is meant to be fast and simple, both during usage and during setup. If you do not understand how it works when you reach here, we failed! Jokes aside, you should not need any consulting service for it.
+Usually hot data are fresh and warm data are old. SeaweedFS puts the newly created volumes on local servers, and optionally upload the older volumes on the cloud. If the older data are accessed less often, this literally gives you unlimited capacity with limited local servers, and still fast for new data. 
 
-SeaweedFS has a centralized master to lookup free volumes, while Ceph uses hashing to locate its objects. Having a centralized master makes it easy to code and manage. HDFS/GFS has the single name node for years. SeaweedFS now support multiple master nodes.
+With the O(1) access time, the network latency cost is kept at minimum. 
 
-Ceph hashing avoids SPOF, but makes it complicated when moving or adding servers.
+If the hot~warm data is split as 20~80, with 20 servers, you can achieve storage capacity of 100 servers. That's a cost saving of 80%! Or you can repurpose the 80 servers to store new data also, and get 5X storage throughput.
+
+[Back to TOC](#table-of-contents)
+
+## Compared to Other File Systems ##
+
+Most other distributed file systems seem more complicated than necessary.
+
+SeaweedFS is meant to be fast and simple, in both setup and operation. If you do not understand how it works when you reach here, we've failed! Please raise an issue with any questions or update this file with clarifications.
+
+[Back to TOC](#table-of-contents)
 
 ### Compared to HDFS ###
-HDFS uses the chunk approach for each file, and is ideal for streaming large files.
+
+HDFS uses the chunk approach for each file, and is ideal for storing large files.
 
 SeaweedFS is ideal for serving relatively smaller files quickly and concurrently.
 
 SeaweedFS can also store extra large files by splitting them into manageable data chunks, and store the file ids of the data chunks into a meta chunk. This is managed by "weed upload/download" tool, and the weed master or volume servers are agnostic about it.
 
-### Compared to MogileFS###
-SeaweedFS has 2 components: directory server, storage nodes.
+[Back to TOC](#table-of-contents)
 
-MogileFS has 3 components: tracers, database, storage nodes.
+### Compared to GlusterFS, Ceph ###
 
-One more layer means slower access, more operation complexity, more failure possibility.
+The architectures are mostly the same. SeaweedFS aims to store and read files fast, with a simple and flat architecture. The main differences are
+
+* SeaweedFS optimizes for small files, ensuring O(1) disk seek operation, and can also handle large files.
+* SeaweedFS statically assigns a volume id for a file. Locating file content becomes just a lookup of the volume id, which can be easily cached.
+* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Cassandra, Redis, Etcd, MySql, Postgres, MemSql, TiDB, CockroachDB, etc, and is easy to customized.
+* SeaweedFS Volume server also communicates directly with clients via HTTP, supporting range queries, direct uploads, etc.
+
+| System         | File Meta                       | File Content Read| POSIX  | REST API | Optimized for small files |
+| -------------  | ------------------------------- | ---------------- | ------ | -------- | ------------------------- |
+| SeaweedFS      | lookup volume id, cacheable     | O(1) disk seek   |        | Yes      | Yes                       |
+| SeaweedFS Filer| Linearly Scalable, Customizable | O(1) disk seek   | FUSE   | Yes      | Yes                       |
+| GlusterFS      | hashing          |                  | FUSE, NFS          |          |                           |
+| Ceph           | hashing + rules  |                  | FUSE               | Yes      |                           |
+
+[Back to TOC](#table-of-contents)
 
 ### Compared to GlusterFS ###
-SeaweedFS is not POSIX compliant, and has simple implementation.
 
-GlusterFS is POSIX compliant, much more complex.
+GlusterFS stores files, both directories and content, in configurable volumes called "bricks".
 
-### Compared to Mongo's GridFS ###
-Mongo's GridFS splits files into chunks and manage chunks in the central mongodb. For every read or write request, the database needs to query the metadata. It's OK if this is not a bottleneck yet, but for a lot of concurrent reads this unnecessary query could slow things down.
+GlusterFS hashes the path and filename into ids, and assigned to virtual volumes, and then mapped to "bricks".
 
-Since files are chunked(default to 256KB), there will be multiple metadata readings and multiple chunk readings, linear to the file size. One  2.56MB file would require at least 20 disk read requests.
+[Back to TOC](#table-of-contents)
 
-On the contrary, SeaweedFS uses large file volume of 32G size to store lots of files, and only manages file volumes in the master server. Each volume manages file metadata themselves. So all the file metadata is spread onto the volume nodes memories, and just one disk read is needed.
+### Compared to Ceph ###
 
-## Dev plan ##
+Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/chrislusf/seaweedfs/issues/120)
+
+SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage.
+
+Same as SeaweedFS, Ceph is also based on the object store RADOS. Ceph is rather complicated with mixed reviews.
+
+Ceph uses CRUSH hashing to automatically manage the data placement. SeaweedFS places data by assigned volumes.
+
+SeaweedFS is optimized for small files. Small files are stored as one continuous block of content, with at most 8 unused bytes between files. Small file access is O(1) disk read.
+
+SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Redis, Etcd, Cassandra, MemSql, TiDB, CockroachCB, to manage file directories. These stores are proven, scalable, and easier to manage.
+
+| SeaweedFS         | comparable to Ceph | advantage |
+| -------------  | ------------- | ---------------- |
+| Master  | MDS | simpler |
+| Volume  | OSD | optimized for small files |
+| Filer  | Ceph FS | linearly scalable, Customizable, O(1) or O(logN) |
+
+[Back to TOC](#table-of-contents)
+
+## Dev Plan ##
 
 More tools and documentation, on how to maintain and scale the system. For example, how to move volumes, automatically balancing data, how to grow volumes, how to check system status, etc.
+Other key features include: Erasure Encoding, JWT security.
 
-This is a super exciting project! And I need helpers!
+This is a super exciting project! And we need helpers and [support](https://www.patreon.com/seaweedfs)!
 
+BTW, We suggest run the code style check script `util/gostd` before you push your branch to remote, it will make SeaweedFS easy to review, maintain and develop:
 
-## Installation guide for users who are not familiar with golang
+```
+$ ./util/gostd
+```
 
-step 1: install go on your machine and setup the environment by following the instructions from the following link:
+[Back to TOC](#table-of-contents)
+
+## Installation Guide ##
+
+> Installation guide for users who are not familiar with golang
+
+Step 1: install go on your machine and setup the environment by following the instructions at:
 
 https://golang.org/doc/install
 
 make sure you set up your $GOPATH
 
 
-step 2: also you may need to install Mercurial by following the instructions below
+Step 2: also you may need to install Mercurial by following the instructions at:
 
 http://mercurial.selenic.com/downloads
 
+Step 3: download, compile, and install the project by executing the following command
 
-step 3: download, compile, and install the project by executing the following command
+```bash
+go get github.com/chrislusf/seaweedfs/weed
+```
 
-go get github.com/chrislusf/weed-fs/go/weed
+Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
 
-once this is done, you should see the executable "weed" under $GOPATH/bin
+Step 4: after you modify your code locally, you could start a local build by calling `go install` under
 
-step 4: after you modify your code locally, you could start a local build by calling "go install" under $GOPATH/src/github.com/chrislusf/weed-fs/go/weed
+```
+$GOPATH/src/github.com/chrislusf/seaweedfs/weed
+```
 
-## Reference
+[Back to TOC](#table-of-contents)
 
-For pre-compiled releases,
- https://bintray.com/chrislusf/Weed-FS/seaweed
-
-## Disk Related topics ##
+## Disk Related Topics ##
 
 ### Hard Drive Performance ###
-When testing read performance on SeaweedFS, it basically becomes performance test your hard drive's random read speed. Hard Drive usually get 100MB/s~200MB/s.
 
-### Solid State Disk
+When testing read performance on SeaweedFS, it basically becomes a performance test of your hard drive's random read speed. Hard drives usually get 100MB/s~200MB/s.
+
+### Solid State Disk ###
 
 To modify or delete small files, SSD must delete a whole block at a time, and move content in existing blocks to a new block. SSD is fast when brand new, but will get fragmented over time and you have to garbage collect, compacting blocks. SeaweedFS is friendly to SSD since it is append-only. Deletion and compaction are done on volume level in the background, not slowing reading and not causing fragmentation.
 
-## Not Planned
+[Back to TOC](#table-of-contents)
 
-POSIX support
+## Benchmark ##
 
-
-## Benchmark
-
-My Own Unscientific Single Machine Results on Mac Book with Solid State Disk, CPU: 1 Intel Core i7 2.2GHz.
+My Own Unscientific Single Machine Results on Mac Book with Solid State Disk, CPU: 1 Intel Core i7 2.6GHz.
 
 Write 1 million 1KB file:
 ```
-Concurrency Level:      64
-Time taken for tests:   182.456 seconds
+Concurrency Level:      16
+Time taken for tests:   66.753 seconds
 Complete requests:      1048576
 Failed requests:        0
-Total transferred:      1073741824 bytes
-Requests per second:    5747.01 [#/sec]
-Transfer rate:          5747.01 [Kbytes/sec]
+Total transferred:      1106789009 bytes
+Requests per second:    15708.23 [#/sec]
+Transfer rate:          16191.69 [Kbytes/sec]
 
 Connection Times (ms)
               min      avg        max      std
-Total:        0.3      10.9       430.9      5.7
+Total:        0.3      1.0       84.3      0.9
 
 Percentage of the requests served within a certain time (ms)
-   50%     10.2 ms
-   66%     12.0 ms
-   75%     12.6 ms
-   80%     12.9 ms
-   90%     14.0 ms
-   95%     14.9 ms
-   98%     16.2 ms
-   99%     17.3 ms
-  100%    430.9 ms
+   50%      0.8 ms
+   66%      1.0 ms
+   75%      1.1 ms
+   80%      1.2 ms
+   90%      1.4 ms
+   95%      1.7 ms
+   98%      2.1 ms
+   99%      2.6 ms
+  100%     84.3 ms
 ```
 
 Randomly read 1 million files:
 ```
-Concurrency Level:      64
-Time taken for tests:   80.732 seconds
+Concurrency Level:      16
+Time taken for tests:   22.301 seconds
 Complete requests:      1048576
 Failed requests:        0
-Total transferred:      1073741824 bytes
-Requests per second:    12988.37 [#/sec]
-Transfer rate:          12988.37 [Kbytes/sec]
+Total transferred:      1106812873 bytes
+Requests per second:    47019.38 [#/sec]
+Transfer rate:          48467.57 [Kbytes/sec]
 
 Connection Times (ms)
               min      avg        max      std
-Total:        0.0      4.7       254.3      6.3
+Total:        0.0      0.3       54.1      0.2
 
 Percentage of the requests served within a certain time (ms)
-   50%      2.6 ms
-   66%      2.9 ms
-   75%      3.7 ms
-   80%      4.7 ms
-   90%     10.3 ms
-   95%     16.6 ms
-   98%     26.3 ms
-   99%     34.8 ms
-  100%    254.3 ms
+   50%      0.3 ms
+   90%      0.4 ms
+   98%      0.6 ms
+   99%      0.7 ms
+  100%     54.1 ms
 ```
 
+[Back to TOC](#table-of-contents)
 
-## License
+## License ##
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -376,3 +528,11 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+
+The text of this page is available for modification and reuse under the terms of the Creative Commons Attribution-Sharealike 3.0 Unported License and the GNU Free Documentation License (unversioned, with no invariant sections, front-cover texts, or back-cover texts).
+
+[Back to TOC](#table-of-contents)
+
+## Stargazers over time ##
+
+[![Stargazers over time](https://starcharts.herokuapp.com/chrislusf/seaweedfs.svg)](https://starcharts.herokuapp.com/chrislusf/seaweedfs)

backers.md

@@ -0,0 +1,13 @@
+<h1 align="center">Sponsors &amp; Backers</h1>
+
+- [Become a backer or sponsor on Patreon](https://www.patreon.com/seaweedfs).
+
+<h2 align="center">Generous Backers ($50+)</h2>
+
+- [4Sight Imaging](https://www.4sightimaging.com/)
+- [Evercam Camera Management Software](https://evercam.io/)
+
+<h2 align="center">Backers</h2>
+
+- [ColorfulClouds Tech Co. Ltd.](https://caiyunai.com/)
+- [Haravan - Ecommerce Platform](https://www.haravan.com)

docker/Dockerfile

@@ -0,0 +1,44 @@
+FROM frolvlad/alpine-glibc
+
+# Supercronic install settings
+ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.8/supercronic-linux-amd64 \
+    SUPERCRONIC=supercronic-linux-amd64 \
+    SUPERCRONIC_SHA1SUM=be43e64c45acd6ec4fce5831e03759c89676a0ea
+
+# Install SeaweedFS and Supercronic ( for cron job mode )
+# Tried to use curl only (curl -o /tmp/linux_amd64.tar.gz ...), however it turned out that the following tar command failed with "gzip: stdin: not in gzip format"
+RUN apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
+    wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o 'chrislusf/seaweedfs/releases/download/.*/linux_amd64.tar.gz') && \
+    tar -C /usr/bin/ -xzvf /tmp/linux_amd64.tar.gz && \
+    curl -fsSLO "$SUPERCRONIC_URL" && \
+    echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
+    chmod +x "$SUPERCRONIC" && \
+    mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
+    ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
+    apk del build-dependencies && \
+    rm -rf /tmp/*
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+
+RUN mkdir -p /data/filerldb2
+
+VOLUME /data
+
+COPY filer.toml /etc/seaweedfs/filer.toml
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/Dockerfile.go_build

@@ -0,0 +1,35 @@
+FROM frolvlad/alpine-glibc as builder
+RUN apk add git go g++
+RUN mkdir -p /go/src/github.com/chrislusf/
+RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
+RUN cd /go/src/github.com/chrislusf/seaweedfs/weed && go install
+
+FROM alpine AS final
+LABEL author="Chris Lu"
+COPY --from=builder /root/go/bin/weed /usr/bin/
+RUN mkdir -p /etc/seaweedfs
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+
+RUN mkdir -p /data/filerldb2
+
+VOLUME /data
+
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/Dockerfile.local

@@ -0,0 +1,29 @@
+FROM alpine AS final
+LABEL author="Chris Lu"
+COPY  ./weed /usr/bin/
+RUN mkdir -p /etc/seaweedfs
+COPY ./filer.toml /etc/seaweedfs/filer.toml
+COPY ./entrypoint.sh /entrypoint.sh
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+
+RUN mkdir -p /data/filerldb2
+
+VOLUME /data
+
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/Makefile

@@ -0,0 +1,19 @@
+all: gen
+
+.PHONY : gen
+
+gen: dev
+
+build:
+	cd ../weed; GOOS=linux go build; mv weed ../docker/
+	docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
+	rm ./weed
+
+dev: build
+	docker-compose -f local-dev-compose.yml -p seaweedfs up
+
+cluster: build
+	docker-compose -f local-cluster-compose.yml -p seaweedfs up
+
+clean:
+	rm ./weed

docker/README.md

@@ -0,0 +1,29 @@
+# Docker
+
+
+## Try it out
+
+```bash
+
+wget https://raw.githubusercontent.com/chrislusf/seaweedfs/master/docker/seaweedfs-compose.yml
+
+docker-compose -f seaweedfs-compose.yml -p seaweedfs up
+
+```
+
+## Try latest tip
+
+```bash
+
+wget https://raw.githubusercontent.com/chrislusf/seaweedfs/master/docker/seaweedfs-dev-compose.yml
+
+docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
+
+```
+
+## Local Development
+
+```bash
+cd $GOPATH/src/github.com/chrislusf/seaweedfs/docker
+make
+```

docker/entrypoint.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+
+case "$1" in
+
+  'master')
+  	ARGS="-mdir /data"
+	# Is this instance linked with an other master? (Docker commandline "--link master1:master")
+	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
+		ARGS="$ARGS -peers=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
+	fi
+  	exec /usr/bin/weed $@ $ARGS
+	;;
+
+  'volume')
+  	ARGS="-ip `hostname -i` -dir /data"
+	# Is this instance linked with a master? (Docker commandline "--link master1:master")
+  	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
+		ARGS="$ARGS -mserver=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
+	fi
+  	exec /usr/bin/weed $@ $ARGS
+	;;
+
+  'server')
+  	ARGS="-ip `hostname -i` -dir /data"
+  	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
+		ARGS="$ARGS -master.peers=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
+	fi
+  	exec /usr/bin/weed $@ $ARGS
+  	;;
+
+  'filer')
+  	ARGS=""
+  	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
+		ARGS="$ARGS -master=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
+	fi
+  	exec /usr/bin/weed $@ $ARGS
+	;;
+
+  's3')
+  	ARGS="-domainName=$S3_DOMAIN_NAME -key.file=$S3_KEY_FILE -cert.file=$S3_CERT_FILE"
+  	if [ -n "$FILER_PORT_8888_TCP_ADDR" ] ; then
+		ARGS="$ARGS -filer=$FILER_PORT_8888_TCP_ADDR:$FILER_PORT_8888_TCP_PORT"
+	fi
+  	exec /usr/bin/weed $@ $ARGS
+	;;
+
+  'cronjob')
+	MASTER=${WEED_MASTER-localhost:9333}
+	FIX_REPLICATION_CRON_SCHEDULE=${CRON_SCHEDULE-*/7 * * * * *}
+	echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "volume.fix.replication" | weed shell -master='$MASTER > /crontab
+	BALANCING_CRON_SCHEDULE=${CRON_SCHEDULE-25 * * * * *}
+	echo "$BALANCING_CRON_SCHEDULE" 'echo "volume.balance -c ALL -force" | weed shell -master='$MASTER >> /crontab
+	echo "Running Crontab:"
+	cat /crontab
+	exec supercronic /crontab
+	;;
+  *)
+  	exec /usr/bin/weed $@
+	;;
+esac

docker/filer.toml

@@ -0,0 +1,3 @@
+[leveldb2]
+enabled = true
+dir = "/data/filerldb2"

docker/local-cluster-compose.yml

@@ -0,0 +1,53 @@
+version: '2'
+
+services:
+  master0:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335"
+  master1:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9334:9334
+      - 19334:19334
+    command: "master -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335"
+  master2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9335:9335
+      - 19335:19335
+    command: "master -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: '-v=2 volume -max=5 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume'
+    depends_on:
+      - master0
+      - master1
+      - master2
+  filer:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+    command: '-v=4 filer -master="master0:9333,master1:9334,master2:9335"'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8333:8333
+    command: '-v=4 s3 -filer="filer:8888"'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume
+      - filer

docker/local-dev-compose.yml

@@ -0,0 +1,35 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: '-v=2 volume -max=5 -mserver="master:9333" -port=8080 -ip=volume'
+    depends_on:
+      - master
+  filer:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+    command: '-v=4 filer -master="master:9333"'
+    depends_on:
+      - master
+      - volume
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8333:8333
+    command: '-v=4 s3 -filer="filer:8888"'
+    depends_on:
+      - master
+      - volume
+      - filer

docker/seaweedfs-compose.yml

@@ -0,0 +1,47 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master"
+  volume:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: 'volume -max=15 -mserver="master:9333" -port=8080'
+    depends_on:
+      - master
+  filer:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+      - 8888:8888
+      - 18888:18888
+    command: 'filer -master="master:9333"'
+    tty: true
+    stdin_open: true
+    depends_on:
+      - master
+      - volume
+  cronjob:
+    image: chrislusf/seaweedfs # use a remote image
+    command: 'cronjob'
+    environment:
+      # Run re-replication every 2 minutes
+      CRON_SCHEDULE: '*/2 * * * * *' # Default: '*/5 * * * * *'
+      WEED_MASTER: master:9333 # Default: localhost:9333
+    depends_on:
+      - master
+      - volume
+  s3:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+      - 8333:8333
+    command: 's3 -filer="filer:8888"'
+    depends_on:
+      - master
+      - volume
+      - filer

docker/seaweedfs-dev-compose.yml

@@ -0,0 +1,35 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:dev # use a remote dev image
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master"
+  volume:
+    image: chrislusf/seaweedfs:dev # use a remote dev image
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: '-v=2 volume -max=5 -mserver="master:9333" -port=8080 -ip=volume'
+    depends_on:
+      - master
+  filer:
+    image: chrislusf/seaweedfs:dev # use a remote dev image
+    ports:
+      - 8888:8888
+      - 18888:18888
+    command: '-v=4 filer -master="master:9333"'
+    depends_on:
+      - master
+      - volume
+  s3:
+    image: chrislusf/seaweedfs:dev # use a remote dev image
+    ports:
+      - 8333:8333
+    command: '-v=4 s3 -filer="filer:8888"'
+    depends_on:
+      - master
+      - volume
+      - filer

go.mod

@@ -0,0 +1,101 @@
+module github.com/chrislusf/seaweedfs
+
+go 1.12
+
+require (
+	cloud.google.com/go v0.44.3
+	github.com/Azure/azure-pipeline-go v0.2.2 // indirect
+	github.com/Azure/azure-storage-blob-go v0.8.0
+	github.com/DataDog/zstd v1.4.1 // indirect
+	github.com/Shopify/sarama v1.23.1
+	github.com/aws/aws-sdk-go v1.23.13
+	github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92
+	github.com/coreos/bbolt v1.3.3 // indirect
+	github.com/coreos/etcd v3.3.15+incompatible // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/disintegration/imaging v1.6.1
+	github.com/dustin/go-humanize v1.0.0
+	github.com/eapache/go-resiliency v1.2.0 // indirect
+	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a
+	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
+	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
+	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
+	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
+	github.com/frankban/quicktest v1.7.2 // indirect
+	github.com/gabriel-vasile/mimetype v1.0.0
+	github.com/go-redis/redis v6.15.2+incompatible
+	github.com/go-sql-driver/mysql v1.4.1
+	github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6
+	github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
+	github.com/golang/protobuf v1.3.2
+	github.com/google/btree v1.0.0
+	github.com/google/uuid v1.1.1
+	github.com/gorilla/mux v1.7.3
+	github.com/gorilla/websocket v1.4.1 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.3 // indirect
+	github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f
+	github.com/jcmturner/gofork v1.0.0 // indirect
+	github.com/karlseguin/ccache v2.0.3+incompatible
+	github.com/karlseguin/expect v1.0.1 // indirect
+	github.com/klauspost/cpuid v1.2.1 // indirect
+	github.com/klauspost/crc32 v1.2.0
+	github.com/klauspost/reedsolomon v1.9.2
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/kurin/blazer v0.5.3
+	github.com/lib/pq v1.2.0
+	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect
+	github.com/mattn/go-runewidth v0.0.4 // indirect
+	github.com/nats-io/nats-server/v2 v2.0.4 // indirect
+	github.com/onsi/ginkgo v1.10.1 // indirect
+	github.com/onsi/gomega v1.7.0 // indirect
+	github.com/pelletier/go-toml v1.4.0 // indirect
+	github.com/peterh/liner v1.1.0
+	github.com/pierrec/lz4 v2.2.7+incompatible // indirect
+	github.com/prometheus/client_golang v1.1.0
+	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
+	github.com/prometheus/procfs v0.0.4 // indirect
+	github.com/rakyll/statik v0.1.6
+	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
+	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
+	github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff
+	github.com/sirupsen/logrus v1.4.2 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/spf13/afero v1.2.2 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/viper v1.4.0
+	github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 // indirect
+	github.com/syndtr/goleveldb v1.0.0
+	github.com/tidwall/gjson v1.3.2
+	github.com/tidwall/match v1.0.1
+	github.com/willf/bitset v1.1.10 // indirect
+	github.com/willf/bloom v2.0.3+incompatible
+	github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 // indirect
+	go.etcd.io/bbolt v1.3.3 // indirect
+	go.etcd.io/etcd v3.3.15+incompatible
+	go.uber.org/multierr v1.2.0 // indirect
+	gocloud.dev v0.16.0
+	gocloud.dev/pubsub/natspubsub v0.16.0
+	gocloud.dev/pubsub/rabbitpubsub v0.16.0
+	golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 // indirect
+	golang.org/x/image v0.0.0-20190829233526-b3c06291d021 // indirect
+	golang.org/x/net v0.0.0-20190909003024-a7b16738d86b
+	golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b
+	golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110
+	google.golang.org/api v0.9.0
+	google.golang.org/appengine v1.6.2 // indirect
+	google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514 // indirect
+	google.golang.org/grpc v1.26.0
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
+	gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
+	gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect
+	sigs.k8s.io/yaml v1.1.0 // indirect
+)
+
+replace github.com/satori/go.uuid v1.2.0 => github.com/satori/go.uuid v0.0.0-20181028125025-b2ce2384e17b

go.sum

@@ -0,0 +1,622 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.3 h1:0sMegbmn/8uTwpNkB0q9cLEpZ2W5a6kl+wtBQgPWBJQ=
+cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA=
+contrib.go.opencensus.io/exporter/ocagent v0.5.0 h1:TKXjQSRS0/cCDrP7KvkgU6SmILtF/yV2TOs/02K/WZQ=
+contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0=
+contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw=
+contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE=
+contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA=
+github.com/Azure/azure-amqp-common-go/v2 v2.1.0/go.mod h1:R8rea+gJRuJR6QxTir/XuEd+YuKoUiazDC/N96FiDEU=
+github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
+github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
+github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo=
+github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
+github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY=
+github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc=
+github.com/Azure/azure-sdk-for-go v29.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-sdk-for-go v30.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
+github.com/Azure/azure-service-bus-go v0.9.1/go.mod h1:yzBx6/BUGfjfeqbRZny9AQIbIe3AcV9WZbAdpkoXOa0=
+github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
+github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o=
+github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0=
+github.com/Azure/go-autorest v12.0.0+incompatible h1:N+VqClcomLGD/sHb3smbSYYtNMgKpVV3Cd5r5i8z6bQ=
+github.com/Azure/go-autorest v12.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 h1:2T/jmrHeTezcCM58lvEQXs0UpQJCo5SoGAcg+mbSTIg=
+github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
+github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
+github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
+github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190605020000-c4ba1fdf4d36/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/Shopify/sarama v1.23.1 h1:XxJBCZEoWJtoWjf/xRbmGUpAmTZGnuuF0ON0EvxxBrs=
+github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs=
+github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
+github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
+github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.19.45/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.23.13 h1:l/NG+mgQFRGG3dsFzEj0jw9JIs/zYdtU6MXhY1WIDmM=
+github.com/aws/aws-sdk-go v1.23.13/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY=
+github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4=
+github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92 h1:lM9SFsh0EPXkyJyrTJqLZPAIJBtNFP6LNkYXu2MnSZI=
+github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92/go.mod h1:4jyiUCD5y548+yKW+oiHtccBiMaLCCbFBpK2t7X4eUo=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY=
+github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.15+incompatible h1:+9RjdC18gMxNQVvSiXvObLu29mOFmkgdsB4cRTlV+EE=
+github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c=
+github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
+github.com/disintegration/imaging v1.6.1 h1:JnBbK6ECIZb1NsWIikP9pd8gIlTIRx7fuDNpU9fsxOE=
+github.com/disintegration/imaging v1.6.1/go.mod h1:xuIt+sRxDFrHS0drzXUlCJthkJ8k7lkkUojDSR247MQ=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU=
+github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
+github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
+github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
+github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw=
+github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA=
+github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0=
+github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
+github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
+github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
+github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 h1:0YtRCqIZs2+Tz49QuH6cJVw/IFqzo39gEqZ0iYLxD2M=
+github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4/go.mod h1:vsJz7uE339KUCpBXx3JAJzSRH7Uk4iGGyJzR529qDIA=
+github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk=
+github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk=
+github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gabriel-vasile/mimetype v1.0.0 h1:0QKnAQQhG6oOsb4GK7iPlet7RtjHi9us8RF/nXoTxhI=
+github.com/gabriel-vasile/mimetype v1.0.0/go.mod h1:6CDPel/o/3/s4+bp6kIbsWATq8pmgOisOPG40CJa6To=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4=
+github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6 h1:P66kRWyEoIx6URKgAC3ijx9jo9gEid7bEhLQ/Z0G65A=
+github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6/go.mod h1:Q7Sru5153KG8D9zwueuQJB3ccJf9/bIwF/x8b3oKgT8=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 h1:X+zN6RZXsvnrSJaAIQhZezPfAfvsqihKKR8oiLHid34=
+github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-replayers/grpcreplay v0.1.0 h1:eNb1y9rZFmY4ax45uEEECSa8fsxGRU+8Bil52ASAwic=
+github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE=
+github.com/google/go-replayers/httpreplay v0.1.0 h1:AX7FUb4BjrrzNvblr/OlgwrmFiep6soj5K2QSDW7BGk=
+github.com/google/go-replayers/httpreplay v0.1.0/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE=
+github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/wire v0.3.0 h1:imGQZGEVEHpje5056+K+cgdO72p0LQv2xIIFXNGUf60=
+github.com/google/wire v0.3.0/go.mod h1:i1DMg/Lu8Sz5yYl25iOdmc5CT5qusaa+zmRWs16741s=
+github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww=
+github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
+github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0 h1:bM6ZAFZmc/wPFaRDi0d5L7hGEZEx/2u+Tmr2evNHDiI=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.9.2/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.11.0 h1:aT5ISUniaOTErogCQ+4pGoYNBB6rm6Fq3g1v8QwYGas=
+github.com/grpc-ecosystem/grpc-gateway v1.11.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
+github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
+github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
+github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f h1:X+tnaqoCcBgAwSTJtoYW6p0qKiuPyMfofEHEFUf2kdU=
+github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f/go.mod h1:Ip4fOwzCrnDVuluHBd7FXIMb7SHOKfkt9/UDrYSZvqI=
+github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjLBdCp5PRlCFijNjvcYANOZXzCfXwCM=
+github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
+github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
+github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
+github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
+github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
+github.com/karlseguin/expect v1.0.1 h1:z4wy4npwwHSWKjGWH85WNJO42VQhovxTCZDSzhjo8hY=
+github.com/karlseguin/expect v1.0.1/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=
+github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/crc32 v1.2.0 h1:0VuyqOCruD33/lJ/ojXNvzVyl8Zr5zdTmj9l9qLZ86I=
+github.com/klauspost/crc32 v1.2.0/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
+github.com/klauspost/reedsolomon v1.9.2 h1:E9CMS2Pqbv+C7tsrYad4YC9MfhnMVWhMRsTi7U0UB18=
+github.com/klauspost/reedsolomon v1.9.2/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kurin/blazer v0.5.3 h1:SAgYv0TKU0kN/ETfO5ExjNAPyMt2FocO2s/UlCHfjAk=
+github.com/kurin/blazer v0.5.3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
+github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA=
+github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
+github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
+github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb h1:hXqqXzQtJbENrsb+rsIqkVqcg4FUJL0SQFGw08Dgivw=
+github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4=
+github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
+github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nats-io/jwt v0.2.6/go.mod h1:mQxQ0uHQ9FhEVPIcTSKwx2lqZEpXWWcCgA7R6NrWvvY=
+github.com/nats-io/jwt v0.2.14 h1:wA50KvFz/JXGXMHRygTWsRGh/ixxgC5E3kHvmtGLNf4=
+github.com/nats-io/jwt v0.2.14/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
+github.com/nats-io/nats-server/v2 v2.0.0/go.mod h1:RyVdsHHvY4B6c9pWG+uRLpZ0h0XsqiuKp2XCTurP5LI=
+github.com/nats-io/nats-server/v2 v2.0.4 h1:XOMeQRbhl1lGNTIctPhih6pTa15NGif54Uas6ZW5q7g=
+github.com/nats-io/nats-server/v2 v2.0.4/go.mod h1:AWdGEVbjKRS9ZIx4DSP5eKW48nfFm7q3uiSkP/1KD7M=
+github.com/nats-io/nats.go v1.8.1 h1:6lF/f1/NN6kzUDBz6pyvQDEXO39jqXcWRLu/tKjtOUQ=
+github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM=
+github.com/nats-io/nkeys v0.0.2 h1:+qM7QpgXnvDDixitZtQUBDY9w/s9mu1ghS+JIbsrx6M=
+github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4=
+github.com/nats-io/nkeys v0.1.0 h1:qMd4+pRHgdr1nAClu+2h/2a5F2TmKcCzjCDazVgRoX4=
+github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
+github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=
+github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/peterh/liner v1.1.0 h1:f+aAedNJA6uk7+6rXsYBnhdo4Xux7ESLe+kcuVUF5os=
+github.com/peterh/liner v1.1.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
+github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
+github.com/pierrec/lz4 v2.2.7+incompatible h1:Eerk9aiqeZo2QzsbWOAsELUf9ddvAxEdMY9LYze/DEc=
+github.com/pierrec/lz4 v2.2.7+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
+github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.4 h1:w8DjqFMJDjuVwdZBQoOozr4MVWOnwF7RcL/7uxBjY78=
+github.com/prometheus/procfs v0.0.4/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rakyll/statik v0.1.6 h1:uICcfUXpgqtw2VopbIncslhAmE5hwc4g20TEyEENBNs=
+github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ=
+github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
+github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc=
+github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
+github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff h1:uLd5zBvf5OA67wcVRePHrFt60bR4LSskaVhgVwyk0Jg=
+github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff/go.mod h1:cubdLmQFqEUZ9vNJrznhgc3m3VMAJi/nY2Ix2axXkG0=
+github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94 h1:0ngsPmuP6XIjiFRNFYlvKwSr5zff2v+uPHaffZ6/M4k=
+github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
+github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 h1:WhxRHzgeVGETMlmVfqhRn8RIeeNoPr2Czh33I4Zdccw=
+github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
+github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/tidwall/gjson v1.3.2 h1:+7p3qQFaH3fOMXAJSrdZwGKcOO/lYdGS0HqGhPqDdTI=
+github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
+github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
+github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA=
+github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8=
+github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
+github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/etcd v3.3.15+incompatible h1:0VpOVCF6EFnJptt8Jh0EWEHO4j2fepyV1fpu9xz/UoQ=
+go.etcd.io/etcd v3.3.15+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
+go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.2.0 h1:6I+W7f5VwC5SV9dNrZ3qXrDB9mD0dyGOi/ZJmYw03T4=
+go.uber.org/multierr v1.2.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+gocloud.dev v0.16.0 h1:hWeaQWxamGerwsU7B9xSWvUjx0p7TwG8fcHro2TzbbM=
+gocloud.dev v0.16.0/go.mod h1:xWGXD8t7bEhqPIuyAUFyXV9qHn+PvEY2F2GKPh7i/O0=
+gocloud.dev/pubsub/natspubsub v0.16.0 h1:MoBGXULDzb1fVaZsGWO5cUCgr6yoI/DHhau8OPGaGEI=
+gocloud.dev/pubsub/natspubsub v0.16.0/go.mod h1:0n7pT7PkLMClBUHDrOkHfOFVr/o/6kawNMwsyAbwadI=
+gocloud.dev/pubsub/rabbitpubsub v0.16.0 h1:Bkv2njMSl2tmT3tGbvbwpiIDAXBIpqzP9dmts+rhD4E=
+gocloud.dev/pubsub/rabbitpubsub v0.16.0/go.mod h1:JJVdUUIqwgaaMJg/1xHQza0g4sI/4KHHSNiGE+pn4JM=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067 h1:KYGJGHOQy8oSi1fDlSpcZF0+juKwk/hEMv5SiwHogR0=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190829233526-b3c06291d021 h1:j6QOxNFMpEL1wIQX6TUdBPNfGZKmBOJS/vfSm8a7tdM=
+golang.org/x/image v0.0.0-20190829233526-b3c06291d021/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc=
+golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190620070143-6f217b454f45/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b h1:3S2h5FadpNr0zUUCVZjlKIEYF+KaX/OBplTGo89CYHI=
+golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110 h1:6S6bidS7O4yAwA5ORRbRIjvNQ9tGbLd5e+LRIaTeVDQ=
+golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
+google.golang.org/api v0.7.0 h1:9sdfJOzWlkqPltHAuzT2Cp+yrBeY1KRVYgms8soxMwM=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0 h1:jbyannxz0XFD3zdjgrSUsaJbgpH4eTrkdhRChkHPfO8=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.2 h1:j8RI1yW0SkI+paT6uGwMlrMI/6zwYA6/CFil8rxOzGI=
+google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514 h1:oFSK4421fpCKRrpzIpybyBVWyht05NegY9+L/3TLAZs=
+google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw=
+gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
+gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=
+gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=
+gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI=
+gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4=
+gopkg.in/jcmturner/gokrb5.v7 v7.2.3 h1:hHMV/yKPwMnJhPuPx7pH2Uw/3Qyf+thJYlisUc44010=
+gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
+gopkg.in/jcmturner/gokrb5.v7 v7.3.0 h1:0709Jtq/6QXEuWRfAm260XqlpcwL1vxtO1tUE2qK8Z4=
+gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
+gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
+gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
+gopkg.in/karlseguin/expect.v1 v1.0.1 h1:9u0iUltnhFbJTHaSIH0EP+cuTU5rafIgmcsEsg2JQFw=
+gopkg.in/karlseguin/expect.v1 v1.0.1/go.mod h1:uB7QIJBcclvYbwlUDkSCsGjAOMis3fP280LyhuDEf2I=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

go/compress/compression_test.go

@@ -1,45 +0,0 @@
-package compress
-
-import (
-	"math/rand"
-	"testing"
-)
-
-func TestSortedData(t *testing.T) {
-	data := make([]int32, 102400)
-	for i := 1; i < len(data); i++ {
-		data[i] = data[i-1] + rand.Int31n(15)
-	}
-	testCompressAndUncompress(t, data, "Sorted data")
-}
-
-func TestUnsortedData(t *testing.T) {
-	data := make([]int32, 102400)
-	for i := 0; i < len(data); i++ {
-		data[i] = rand.Int31n(255)
-	}
-	testCompressAndUncompress(t, data, "Unsorted data")
-}
-
-func testCompressAndUncompress(t *testing.T, data []int32, desc string) {
-
-	compressed_data, err := Compress32(data)
-	if err != nil {
-		t.Fatal("Compress error", err.Error())
-	}
-	uncompressed_data, err := Uncompress32(compressed_data, make([]int32, len(data)*2))
-	if err != nil {
-		t.Fatal("Compress error", err.Error())
-	}
-	if len(uncompressed_data) != len(data) {
-		t.Fatal("Len differs", len(data), len(uncompressed_data))
-	}
-	for i := 0; i < len(data); i++ {
-		if data[i] != uncompressed_data[i] {
-			t.Fatal("Data differs:", i, data[i], uncompressed_data[i])
-		}
-	}
-
-	println(desc, " Data length:", len(data), " => Compressed length:", len(compressed_data))
-
-}

go/compress/delta_binary_pack32.go

@@ -1,32 +0,0 @@
-package compress
-
-import (
-	"github.com/reducedb/encoding/cursor"
-	"github.com/reducedb/encoding/delta/bp32"
-)
-
-// Compress compresses in[]int32 to out[]int32
-func Compress32(in []int32) (out []int32, err error) {
-	out = make([]int32, len(in)*2)
-	inpos := cursor.New()
-	outpos := cursor.New()
-
-	if err = bp32.New().Compress(in, inpos, len(in), out, outpos); err != nil {
-		return nil, err
-	}
-
-	return out[:outpos.Get()], nil
-}
-
-// Uncompress uncompresses in[]int32 to out[]int32
-func Uncompress32(in []int32, buffer []int32) (out []int32, err error) {
-	out = buffer
-	inpos := cursor.New()
-	outpos := cursor.New()
-
-	if err = bp32.New().Uncompress(in, inpos, len(in), out, outpos); err != nil {
-		return nil, err
-	}
-
-	return out[:outpos.Get()], nil
-}

go/filer/cassandra_store/cassandra_store.go

@@ -1,87 +0,0 @@
-package cassandra_store
-
-import (
-	"fmt"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-
-	"github.com/gocql/gocql"
-)
-
-/*
-
-Basically you need a table just like this:
-
-CREATE TABLE seaweed_files (
-   path varchar,
-   fids list<varchar>,
-   PRIMARY KEY (path)
-);
-
-Need to match flat_namespace.FlatNamespaceStore interface
-	Put(fullFileName string, fid string) (err error)
-	Get(fullFileName string) (fid string, err error)
-	Delete(fullFileName string) (fid string, err error)
-
-*/
-type CassandraStore struct {
-	cluster *gocql.ClusterConfig
-	session *gocql.Session
-}
-
-func NewCassandraStore(keyspace string, hosts ...string) (c *CassandraStore, err error) {
-	c = &CassandraStore{}
-	c.cluster = gocql.NewCluster(hosts...)
-	c.cluster.Keyspace = keyspace
-	c.cluster.Consistency = gocql.Quorum
-	c.session, err = c.cluster.CreateSession()
-	if err != nil {
-		glog.V(0).Infof("Failed to open cassandra store, hosts %v, keyspace %s", hosts, keyspace)
-	}
-	return
-}
-
-func (c *CassandraStore) Put(fullFileName string, fid string) (err error) {
-	var input []string
-	input = append(input, fid)
-	if err := c.session.Query(
-		`INSERT INTO seaweed_files (path, fids) VALUES (?, ?)`,
-		fullFileName, input).Exec(); err != nil {
-		glog.V(0).Infof("Failed to save file %s with id %s: %v", fullFileName, fid, err)
-		return err
-	}
-	return nil
-}
-func (c *CassandraStore) Get(fullFileName string) (fid string, err error) {
-	var output []string
-	if err := c.session.Query(
-		`select fids FROM seaweed_files WHERE path = ? LIMIT 1`,
-		fullFileName).Consistency(gocql.One).Scan(&output); err != nil {
-		if err != gocql.ErrNotFound {
-			glog.V(0).Infof("Failed to find file %s: %v", fullFileName, fid, err)
-		}
-	}
-	if len(output) == 0 {
-		return "", fmt.Errorf("No file id found for %s", fullFileName)
-	}
-	return output[0], nil
-}
-
-// Currently the fid is not returned
-func (c *CassandraStore) Delete(fullFileName string) (fid string, err error) {
-	if err := c.session.Query(
-		`DELETE FROM seaweed_files WHERE path = ?`,
-		fullFileName).Exec(); err != nil {
-		if err != gocql.ErrNotFound {
-			glog.V(0).Infof("Failed to delete file %s: %v", fullFileName, err)
-		}
-		return "", err
-	}
-	return "", nil
-}
-
-func (c *CassandraStore) Close() {
-	if c.session != nil {
-		c.session.Close()
-	}
-}

go/filer/cassandra_store/schema.cql

@@ -1,22 +0,0 @@
-/*
-
-Here is the CQL to create the table.CassandraStore
-
-Optionally you can adjust the keyspace name and replication settings.
-
-For production server, very likely you want to set replication_factor to 3
-
-*/
-
-create keyspace seaweed WITH replication = {
-  'class':'SimpleStrategy',
-  'replication_factor':1
-};
-
-use seaweed;
-
-CREATE TABLE seaweed_files (
-   path varchar,
-   fids list<varchar>,
-   PRIMARY KEY (path)
-);

go/filer/client_operations.go

@@ -1,70 +0,0 @@
-package filer
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-
-	"github.com/chrislusf/weed-fs/go/util"
-
-	"net/url"
-)
-
-type ApiRequest struct {
-	Command   string //"listFiles", "listDirectories"
-	Directory string
-	FileName  string
-}
-
-type ListFilesResult struct {
-	Files []FileEntry
-	Error string `json:"error,omitempty"`
-}
-
-func ListFiles(server string, directory string, fileName string) (*ListFilesResult, error) {
-	var ret ListFilesResult
-	if err := call(server, ApiRequest{Command: "listFiles", Directory: directory, FileName: fileName}, &ret); err == nil {
-		if ret.Error != "" {
-			return nil, errors.New(ret.Error)
-		}
-		return &ret, nil
-	} else {
-		return nil, err
-	}
-}
-
-type ListDirectoriesResult struct {
-	Directories []DirectoryEntry
-	Error       string `json:"error,omitempty"`
-}
-
-func ListDirectories(server string, directory string) (*ListDirectoriesResult, error) {
-	var ret ListDirectoriesResult
-	if err := call(server, ApiRequest{Command: "listDirectories", Directory: directory}, &ret); err == nil {
-		if ret.Error != "" {
-			return nil, errors.New(ret.Error)
-		}
-		return &ret, nil
-	} else {
-		return nil, err
-	}
-}
-
-func call(server string, request ApiRequest, ret interface{}) error {
-	b, err := json.Marshal(request)
-	if err != nil {
-		fmt.Println("error:", err)
-		return nil
-	}
-	values := make(url.Values)
-	values.Add("request", string(b))
-	jsonBlob, err := util.Post("http://"+server+"/__api__", values)
-	if err != nil {
-		return err
-	}
-	err = json.Unmarshal(jsonBlob, ret)
-	if err != nil {
-		return err
-	}
-	return nil
-}

go/filer/embedded_filer/design.txt

@@ -1,26 +0,0 @@
-Design Assumptions:
-1. the number of directories are magnitudely smaller than the number of files
-2. unlimited number of files under any directories
-Phylosophy:
-  metadata for directories and files should be separated
-Design:
-  Store directories in normal map
-    all of directories hopefully all be in memory
-    efficient to move/rename/list_directories
-  Log directory changes to append only log file
-  Store files in sorted string table in <dir_id/filename> format
-    efficient to list_files, just simple iterator
-    efficient to locate files, binary search
-
-Testing:
-1. starting server, "weed server -filer=true"
-2. posting files to different folders
-curl -F "filename=@design.txt"   "http://localhost:8888/sources/"
-curl -F "filename=@design.txt"   "http://localhost:8888/design/"
-curl -F "filename=@directory.go" "http://localhost:8888/sources/weed/go/"
-curl -F "filename=@directory.go" "http://localhost:8888/sources/testing/go/"
-curl -F "filename=@filer.go"     "http://localhost:8888/sources/weed/go/"
-curl -F "filename=@filer_in_leveldb.go"     "http://localhost:8888/sources/weed/go/"
-curl "http://localhost:8888/?pretty=y"
-curl "http://localhost:8888/sources/weed/go/?pretty=y"
-curl "http://localhost:8888/sources/weed/go/?pretty=y"

go/filer/embedded_filer/directory.go

@@ -1,15 +0,0 @@
-package embedded_filer
-
-import (
-	"github.com/chrislusf/weed-fs/go/filer"
-)
-
-type DirectoryManager interface {
-	FindDirectory(dirPath string) (filer.DirectoryId, error)
-	ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error)
-	MakeDirectory(currentDirPath string, dirName string) (filer.DirectoryId, error)
-	MoveUnderDirectory(oldDirPath string, newParentDirPath string) error
-	DeleteDirectory(dirPath string) error
-	//functions used by FUSE
-	FindDirectoryById(filer.DirectoryId, error)
-}

go/filer/embedded_filer/directory_in_map.go

@@ -1,259 +0,0 @@
-package embedded_filer
-
-import (
-	"bufio"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"sync"
-
-	"github.com/chrislusf/weed-fs/go/filer"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-var writeLock sync.Mutex //serialize changes to dir.log
-
-type DirectoryEntryInMap struct {
-	Name           string
-	Parent         *DirectoryEntryInMap
-	SubDirectories map[string]*DirectoryEntryInMap
-	Id             filer.DirectoryId
-}
-
-type DirectoryManagerInMap struct {
-	Root      *DirectoryEntryInMap
-	max       filer.DirectoryId
-	logFile   *os.File
-	isLoading bool
-}
-
-func (dm *DirectoryManagerInMap) NewDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap, err error) {
-	writeLock.Lock()
-	defer writeLock.Unlock()
-	d = &DirectoryEntryInMap{Name: name, Parent: parent, SubDirectories: make(map[string]*DirectoryEntryInMap)}
-	var parts []string
-	for p := d; p != nil && p.Name != ""; p = p.Parent {
-		parts = append(parts, p.Name)
-	}
-	n := len(parts)
-	if n <= 0 {
-		return nil, fmt.Errorf("Failed to create folder %s/%s", parent.Name, name)
-	}
-	for i := 0; i < n/2; i++ {
-		parts[i], parts[n-1-i] = parts[n-1-i], parts[i]
-	}
-	dm.max++
-	d.Id = dm.max
-	dm.log("add", "/"+strings.Join(parts, "/"), strconv.Itoa(int(d.Id)))
-	return d, nil
-}
-
-func (dm *DirectoryManagerInMap) log(words ...string) {
-	if !dm.isLoading {
-		dm.logFile.WriteString(strings.Join(words, "\t") + "\n")
-	}
-}
-
-func NewDirectoryManagerInMap(dirLogFile string) (dm *DirectoryManagerInMap, err error) {
-	dm = &DirectoryManagerInMap{}
-	//dm.Root do not use NewDirectoryEntryInMap, since dm.max will be changed
-	dm.Root = &DirectoryEntryInMap{SubDirectories: make(map[string]*DirectoryEntryInMap)}
-	if dm.logFile, err = os.OpenFile(dirLogFile, os.O_RDWR|os.O_CREATE, 0644); err != nil {
-		return nil, fmt.Errorf("cannot write directory log file %s.idx: %v", dirLogFile, err)
-	}
-	return dm, dm.load()
-}
-
-func (dm *DirectoryManagerInMap) processEachLine(line string) error {
-	if strings.HasPrefix(line, "#") {
-		return nil
-	}
-	if line == "" {
-		return nil
-	}
-	parts := strings.Split(line, "\t")
-	if len(parts) == 0 {
-		return nil
-	}
-	switch parts[0] {
-	case "add":
-		v, pe := strconv.Atoi(parts[2])
-		if pe != nil {
-			return pe
-		}
-		if e := dm.loadDirectory(parts[1], filer.DirectoryId(v)); e != nil {
-			return e
-		}
-	case "mov":
-		newName := ""
-		if len(parts) >= 4 {
-			newName = parts[3]
-		}
-		if e := dm.MoveUnderDirectory(parts[1], parts[2], newName); e != nil {
-			return e
-		}
-	case "del":
-		if e := dm.DeleteDirectory(parts[1]); e != nil {
-			return e
-		}
-	default:
-		fmt.Printf("line %s has %s!\n", line, parts[0])
-		return nil
-	}
-	return nil
-}
-func (dm *DirectoryManagerInMap) load() error {
-	dm.max = 0
-	lines := bufio.NewReader(dm.logFile)
-	dm.isLoading = true
-	defer func() { dm.isLoading = false }()
-	for {
-		line, err := util.Readln(lines)
-		if err != nil && err != io.EOF {
-			return err
-		}
-		if pe := dm.processEachLine(string(line)); pe != nil {
-			return pe
-		}
-		if err == io.EOF {
-			return nil
-		}
-	}
-}
-
-func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryInMap, error) {
-	if dirPath == "" {
-		return dm.Root, nil
-	}
-	dirPath = filepath.Clean(dirPath)
-	if dirPath == "/" {
-		return dm.Root, nil
-	}
-	parts := strings.Split(dirPath, "/")
-	dir := dm.Root
-	for i := 1; i < len(parts); i++ {
-		if sub, ok := dir.SubDirectories[parts[i]]; ok {
-			dir = sub
-		} else {
-			return dm.Root, fmt.Errorf("Directory %s Not Found", dirPath)
-		}
-	}
-	return dir, nil
-}
-func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryId, error) {
-	d, e := dm.findDirectory(dirPath)
-	if e == nil {
-		return d.Id, nil
-	}
-	return dm.Root.Id, e
-}
-
-func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error {
-	dirPath = filepath.Clean(dirPath)
-	if dirPath == "/" {
-		return nil
-	}
-	parts := strings.Split(dirPath, "/")
-	dir := dm.Root
-	for i := 1; i < len(parts); i++ {
-		sub, ok := dir.SubDirectories[parts[i]]
-		if !ok {
-			if i != len(parts)-1 {
-				return fmt.Errorf("%s should be created after parent %s", dirPath, parts[i])
-			}
-			var err error
-			sub, err = dm.NewDirectoryEntryInMap(dir, parts[i])
-			if err != nil {
-				return err
-			}
-			if sub.Id != dirId {
-				return fmt.Errorf("%s should be have id %v instead of %v", dirPath, sub.Id, dirId)
-			}
-			dir.SubDirectories[parts[i]] = sub
-		}
-		dir = sub
-	}
-	return nil
-}
-
-func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEntryInMap, created bool) {
-	dirPath = filepath.Clean(dirPath)
-	if dirPath == "/" {
-		return dm.Root, false
-	}
-	parts := strings.Split(dirPath, "/")
-	dir = dm.Root
-	for i := 1; i < len(parts); i++ {
-		sub, ok := dir.SubDirectories[parts[i]]
-		if !ok {
-			var err error
-			sub, err = dm.NewDirectoryEntryInMap(dir, parts[i])
-			if err != nil {
-				return nil, false
-			}
-			dir.SubDirectories[parts[i]] = sub
-			created = true
-		}
-		dir = sub
-	}
-	return dir, created
-}
-
-func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (filer.DirectoryId, error) {
-	dir, _ := dm.makeDirectory(dirPath)
-	return dir.Id, nil
-}
-
-func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParentDirPath string, newName string) error {
-	writeLock.Lock()
-	defer writeLock.Unlock()
-	oldDir, oe := dm.findDirectory(oldDirPath)
-	if oe != nil {
-		return oe
-	}
-	parentDir, pe := dm.findDirectory(newParentDirPath)
-	if pe != nil {
-		return pe
-	}
-	dm.log("mov", oldDirPath, newParentDirPath, newName)
-	delete(oldDir.Parent.SubDirectories, oldDir.Name)
-	if newName == "" {
-		newName = oldDir.Name
-	}
-	parentDir.SubDirectories[newName] = oldDir
-	oldDir.Name = newName
-	oldDir.Parent = parentDir
-	return nil
-}
-
-func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryEntry, err error) {
-	d, e := dm.findDirectory(dirPath)
-	if e != nil {
-		return dirNames, e
-	}
-	for k, v := range d.SubDirectories {
-		dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id})
-	}
-	return dirNames, nil
-}
-func (dm *DirectoryManagerInMap) DeleteDirectory(dirPath string) error {
-	writeLock.Lock()
-	defer writeLock.Unlock()
-	if dirPath == "/" {
-		return fmt.Errorf("Can not delete %s", dirPath)
-	}
-	d, e := dm.findDirectory(dirPath)
-	if e != nil {
-		return e
-	}
-	if len(d.SubDirectories) != 0 {
-		return fmt.Errorf("dir %s still has sub directories", dirPath)
-	}
-	delete(d.Parent.SubDirectories, d.Name)
-	d.Parent = nil
-	dm.log("del", dirPath)
-	return nil
-}

go/filer/embedded_filer/directory_test.go

@@ -1,86 +0,0 @@
-package embedded_filer
-
-import (
-	"os"
-	"strings"
-	"testing"
-)
-
-func TestDirectory(t *testing.T) {
-	dm, _ := NewDirectoryManagerInMap("/tmp/dir.log")
-	defer func() {
-		if true {
-			os.Remove("/tmp/dir.log")
-		}
-	}()
-	dm.MakeDirectory("/a/b/c")
-	dm.MakeDirectory("/a/b/d")
-	dm.MakeDirectory("/a/b/e")
-	dm.MakeDirectory("/a/b/e/f")
-	dm.MakeDirectory("/a/b/e/f/g")
-	dm.MoveUnderDirectory("/a/b/e/f/g", "/a/b", "t")
-	if _, err := dm.FindDirectory("/a/b/e/f/g"); err == nil {
-		t.Fatal("/a/b/e/f/g should not exist any more after moving")
-	}
-	if _, err := dm.FindDirectory("/a/b/t"); err != nil {
-		t.Fatal("/a/b/t should exist after moving")
-	}
-	if _, err := dm.FindDirectory("/a/b/g"); err == nil {
-		t.Fatal("/a/b/g should not exist after moving")
-	}
-	dm.MoveUnderDirectory("/a/b/e/f", "/a/b", "")
-	if _, err := dm.FindDirectory("/a/b/f"); err != nil {
-		t.Fatal("/a/b/g should not exist after moving")
-	}
-	dm.MakeDirectory("/a/b/g/h/i")
-	dm.DeleteDirectory("/a/b/e/f")
-	dm.DeleteDirectory("/a/b/e")
-	dirNames, _ := dm.ListDirectories("/a/b/e")
-	for _, v := range dirNames {
-		println("sub1 dir:", v.Name, "id", v.Id)
-	}
-	dm.logFile.Close()
-
-	var path []string
-	printTree(dm.Root, path)
-
-	dm2, e := NewDirectoryManagerInMap("/tmp/dir.log")
-	if e != nil {
-		println("load error", e.Error())
-	}
-	if !compare(dm.Root, dm2.Root) {
-		t.Fatal("restored dir not the same!")
-	}
-	printTree(dm2.Root, path)
-}
-
-func printTree(node *DirectoryEntryInMap, path []string) {
-	println(strings.Join(path, "/") + "/" + node.Name)
-	path = append(path, node.Name)
-	for _, v := range node.SubDirectories {
-		printTree(v, path)
-	}
-}
-
-func compare(root1 *DirectoryEntryInMap, root2 *DirectoryEntryInMap) bool {
-	if len(root1.SubDirectories) != len(root2.SubDirectories) {
-		return false
-	}
-	if root1.Name != root2.Name {
-		return false
-	}
-	if root1.Id != root2.Id {
-		return false
-	}
-	if !(root1.Parent == nil && root2.Parent == nil) {
-		if root1.Parent.Id != root2.Parent.Id {
-			return false
-		}
-	}
-	for k, v := range root1.SubDirectories {
-		if !compare(v, root2.SubDirectories[k]) {
-			return false
-		}
-	}
-	return true
-}

go/filer/embedded_filer/filer_embedded.go

@@ -1,141 +0,0 @@
-package embedded_filer
-
-import (
-	"errors"
-	"fmt"
-	"path/filepath"
-	"strings"
-
-	"github.com/chrislusf/weed-fs/go/filer"
-	"github.com/chrislusf/weed-fs/go/operation"
-)
-
-type FilerEmbedded struct {
-	master      string
-	directories *DirectoryManagerInMap
-	files       *FileListInLevelDb
-}
-
-func NewFilerEmbedded(master string, dir string) (filer *FilerEmbedded, err error) {
-	dm, de := NewDirectoryManagerInMap(filepath.Join(dir, "dir.log"))
-	if de != nil {
-		return nil, de
-	}
-	fl, fe := NewFileListInLevelDb(dir)
-	if fe != nil {
-		return nil, fe
-	}
-	filer = &FilerEmbedded{
-		master:      master,
-		directories: dm,
-		files:       fl,
-	}
-	return
-}
-
-func (filer *FilerEmbedded) CreateFile(filePath string, fid string) (err error) {
-	dir, file := filepath.Split(filePath)
-	dirId, e := filer.directories.MakeDirectory(dir)
-	if e != nil {
-		return e
-	}
-	return filer.files.CreateFile(dirId, file, fid)
-}
-func (filer *FilerEmbedded) FindFile(filePath string) (fid string, err error) {
-	dir, file := filepath.Split(filePath)
-	dirId, e := filer.directories.FindDirectory(dir)
-	if e != nil {
-		return "", e
-	}
-	return filer.files.FindFile(dirId, file)
-}
-func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
-	return filer.directories.FindDirectory(dirPath)
-}
-func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
-	return filer.directories.ListDirectories(dirPath)
-}
-func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
-	dirId, e := filer.directories.FindDirectory(dirPath)
-	if e != nil {
-		return nil, e
-	}
-	return filer.files.ListFiles(dirId, lastFileName, limit), nil
-}
-func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err error) {
-	dirId, e := filer.directories.FindDirectory(dirPath)
-	if e != nil {
-		return e
-	}
-	if sub_dirs, sub_err := filer.directories.ListDirectories(dirPath); sub_err == nil {
-		if len(sub_dirs) > 0 && !recursive {
-			return fmt.Errorf("Fail to delete directory %s: %d sub directories found!", dirPath, len(sub_dirs))
-		}
-		for _, sub := range sub_dirs {
-			if delete_sub_err := filer.DeleteDirectory(filepath.Join(dirPath, sub.Name), recursive); delete_sub_err != nil {
-				return delete_sub_err
-			}
-		}
-	}
-	list := filer.files.ListFiles(dirId, "", 100)
-	if len(list) != 0 && !recursive {
-		if !recursive {
-			return fmt.Errorf("Fail to delete non-empty directory %s!", dirPath)
-		}
-	}
-	for {
-		if len(list) == 0 {
-			return filer.directories.DeleteDirectory(dirPath)
-		}
-		var fids []string
-		for _, fileEntry := range list {
-			fids = append(fids, string(fileEntry.Id))
-		}
-		if result_list, delete_file_err := operation.DeleteFiles(filer.master, fids); delete_file_err != nil {
-			return delete_file_err
-		} else {
-			if len(result_list.Errors) > 0 {
-				return errors.New(strings.Join(result_list.Errors, "\n"))
-			}
-		}
-		lastFile := list[len(list)-1]
-		list = filer.files.ListFiles(dirId, lastFile.Name, 100)
-	}
-
-}
-
-func (filer *FilerEmbedded) DeleteFile(filePath string) (fid string, err error) {
-	dir, file := filepath.Split(filePath)
-	dirId, e := filer.directories.FindDirectory(dir)
-	if e != nil {
-		return "", e
-	}
-	return filer.files.DeleteFile(dirId, file)
-}
-
-/*
-Move a folder or a file, with 4 Use cases:
-mv fromDir toNewDir
-mv fromDir toOldDir
-mv fromFile toDir
-mv fromFile toFile
-*/
-func (filer *FilerEmbedded) Move(fromPath string, toPath string) error {
-	if _, dir_err := filer.FindDirectory(fromPath); dir_err == nil {
-		if _, err := filer.FindDirectory(toPath); err == nil {
-			// move folder under an existing folder
-			return filer.directories.MoveUnderDirectory(fromPath, toPath, "")
-		}
-		// move folder to a new folder
-		return filer.directories.MoveUnderDirectory(fromPath, filepath.Dir(toPath), filepath.Base(toPath))
-	}
-	if fid, file_err := filer.DeleteFile(fromPath); file_err == nil {
-		if _, err := filer.FindDirectory(toPath); err == nil {
-			// move file under an existing folder
-			return filer.CreateFile(filepath.Join(toPath, filepath.Base(fromPath)), fid)
-		}
-		// move to a folder with new name
-		return filer.CreateFile(toPath, fid)
-	}
-	return fmt.Errorf("File %s is not found!", fromPath)
-}

go/filer/embedded_filer/files_in_leveldb.go

@@ -1,82 +0,0 @@
-package embedded_filer
-
-import (
-	"bytes"
-
-	"github.com/chrislusf/weed-fs/go/filer"
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-/*
-The entry in level db has this format:
-  key: genKey(dirId, fileName)
-  value: []byte(fid)
-And genKey(dirId, fileName) use first 4 bytes to store dirId, and rest for fileName
-*/
-
-type FileListInLevelDb struct {
-	db *leveldb.DB
-}
-
-func NewFileListInLevelDb(dir string) (fl *FileListInLevelDb, err error) {
-	fl = &FileListInLevelDb{}
-	if fl.db, err = leveldb.OpenFile(dir, nil); err != nil {
-		return
-	}
-	return
-}
-
-func genKey(dirId filer.DirectoryId, fileName string) []byte {
-	ret := make([]byte, 0, 4+len(fileName))
-	for i := 3; i >= 0; i-- {
-		ret = append(ret, byte(dirId>>(uint(i)*8)))
-	}
-	ret = append(ret, []byte(fileName)...)
-	return ret
-}
-
-func (fl *FileListInLevelDb) CreateFile(dirId filer.DirectoryId, fileName string, fid string) (err error) {
-	glog.V(4).Infoln("directory", dirId, "fileName", fileName, "fid", fid)
-	return fl.db.Put(genKey(dirId, fileName), []byte(fid), nil)
-}
-func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
-	if fid, err = fl.FindFile(dirId, fileName); err != nil {
-		return
-	}
-	err = fl.db.Delete(genKey(dirId, fileName), nil)
-	return fid, err
-}
-func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
-	data, e := fl.db.Get(genKey(dirId, fileName), nil)
-	if e != nil {
-		return "", e
-	}
-	return string(data), nil
-}
-func (fl *FileListInLevelDb) ListFiles(dirId filer.DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) {
-	glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit)
-	dirKey := genKey(dirId, "")
-	iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil)
-	limitCounter := 0
-	for iter.Next() {
-		key := iter.Key()
-		if !bytes.HasPrefix(key, dirKey) {
-			break
-		}
-		fileName := string(key[len(dirKey):])
-		if fileName == lastFileName {
-			continue
-		}
-		limitCounter++
-		if limit > 0 {
-			if limitCounter > limit {
-				break
-			}
-		}
-		files = append(files, filer.FileEntry{Name: fileName, Id: filer.FileId(string(iter.Value()))})
-	}
-	iter.Release()
-	return
-}

go/filer/filer.go

@@ -1,28 +0,0 @@
-package filer
-
-type FileId string //file id on weedfs
-
-type FileEntry struct {
-	Name string `json:"name,omitempty"` //file name without path
-	Id   FileId `json:"fid,omitempty"`
-}
-
-type DirectoryId int32
-
-type DirectoryEntry struct {
-	Name string //dir name without path
-	Id   DirectoryId
-}
-
-type Filer interface {
-	CreateFile(fullFileName string, fid string) (err error)
-	FindFile(fullFileName string) (fid string, err error)
-	DeleteFile(fullFileName string) (fid string, err error)
-
-	//Optional functions. embedded filer support these
-	FindDirectory(dirPath string) (dirId DirectoryId, err error)
-	ListDirectories(dirPath string) (dirs []DirectoryEntry, err error)
-	ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error)
-	DeleteDirectory(dirPath string, recursive bool) (err error)
-	Move(fromPath string, toPath string) (err error)
-}

go/filer/flat_namespace/flat_namespace_filer.go

@@ -1,50 +0,0 @@
-package flat_namespace
-
-import (
-	"errors"
-
-	"github.com/chrislusf/weed-fs/go/filer"
-)
-
-type FlatNamesapceFiler struct {
-	master string
-	store  FlatNamespaceStore
-}
-
-var (
-	ErrNotImplemented = errors.New("Not Implemented for flat namespace meta data store")
-)
-
-func NewFlatNamesapceFiler(master string, store FlatNamespaceStore) *FlatNamesapceFiler {
-	return &FlatNamesapceFiler{
-		master: master,
-		store:  store,
-	}
-}
-
-func (filer *FlatNamesapceFiler) CreateFile(fullFileName string, fid string) (err error) {
-	return filer.store.Put(fullFileName, fid)
-}
-func (filer *FlatNamesapceFiler) FindFile(fullFileName string) (fid string, err error) {
-	return filer.store.Get(fullFileName)
-}
-func (filer *FlatNamesapceFiler) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
-	return 0, ErrNotImplemented
-}
-func (filer *FlatNamesapceFiler) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
-	return nil, ErrNotImplemented
-}
-func (filer *FlatNamesapceFiler) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
-	return nil, ErrNotImplemented
-}
-func (filer *FlatNamesapceFiler) DeleteDirectory(dirPath string, recursive bool) (err error) {
-	return ErrNotImplemented
-}
-
-func (filer *FlatNamesapceFiler) DeleteFile(fullFileName string) (fid string, err error) {
-	return filer.store.Delete(fullFileName)
-}
-
-func (filer *FlatNamesapceFiler) Move(fromPath string, toPath string) error {
-	return ErrNotImplemented
-}

go/filer/flat_namespace/flat_namespace_store.go

@@ -1,9 +0,0 @@
-package flat_namespace
-
-import ()
-
-type FlatNamespaceStore interface {
-	Put(fullFileName string, fid string) (err error)
-	Get(fullFileName string) (fid string, err error)
-	Delete(fullFileName string) (fid string, err error)
-}

go/filer/redis_store/redis_store.go

@@ -1,48 +0,0 @@
-package redis_store
-
-import (
-	redis "gopkg.in/redis.v2"
-)
-
-type RedisStore struct {
-	Client *redis.Client
-}
-
-func NewRedisStore(hostPort string, database int) *RedisStore {
-	client := redis.NewTCPClient(&redis.Options{
-		Addr:     hostPort,
-		Password: "", // no password set
-		DB:       int64(database),
-	})
-	return &RedisStore{Client: client}
-}
-
-func (s *RedisStore) Get(fullFileName string) (fid string, err error) {
-	fid, err = s.Client.Get(fullFileName).Result()
-	if err == redis.Nil {
-		err = nil
-	}
-	return fid, err
-}
-func (s *RedisStore) Put(fullFileName string, fid string) (err error) {
-	_, err = s.Client.Set(fullFileName, fid).Result()
-	if err == redis.Nil {
-		err = nil
-	}
-	return err
-}
-
-// Currently the fid is not returned
-func (s *RedisStore) Delete(fullFileName string) (fid string, err error) {
-	_, err = s.Client.Del(fullFileName).Result()
-	if err == redis.Nil {
-		err = nil
-	}
-	return "", err
-}
-
-func (s *RedisStore) Close() {
-	if s.Client != nil {
-		s.Client.Close()
-	}
-}

go/images/resizing.go

@@ -1,46 +0,0 @@
-package images
-
-import (
-	"bytes"
-	"image"
-	"image/gif"
-	"image/jpeg"
-	"image/png"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/disintegration/imaging"
-)
-
-func Resized(ext string, data []byte, width, height int) (resized []byte, w int, h int) {
-	if width == 0 && height == 0 {
-		return data, 0, 0
-	}
-	srcImage, _, err := image.Decode(bytes.NewReader(data))
-	if err == nil {
-		bounds := srcImage.Bounds()
-		var dstImage *image.NRGBA
-		if bounds.Dx() > width && width != 0 || bounds.Dy() > height && height != 0 {
-			if width == height && bounds.Dx() != bounds.Dy() {
-				dstImage = imaging.Thumbnail(srcImage, width, height, imaging.Lanczos)
-				w, h = width, height
-			} else {
-				dstImage = imaging.Resize(srcImage, width, height, imaging.Lanczos)
-			}
-		} else {
-			return data, bounds.Dx(), bounds.Dy()
-		}
-		var buf bytes.Buffer
-		switch ext {
-		case ".png":
-			png.Encode(&buf, dstImage)
-		case ".jpg", ".jpeg":
-			jpeg.Encode(&buf, dstImage, nil)
-		case ".gif":
-			gif.Encode(&buf, dstImage, nil)
-		}
-		return buf.Bytes(), dstImage.Bounds().Dx(), dstImage.Bounds().Dy()
-	} else {
-		glog.Error(err)
-	}
-	return data, 0, 0
-}

go/operation/assign_file_id.go

@@ -1,48 +0,0 @@
-package operation
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net/url"
-	"strconv"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-type AssignResult struct {
-	Fid       string `json:"fid,omitempty"`
-	Url       string `json:"url,omitempty"`
-	PublicUrl string `json:"publicUrl,omitempty"`
-	Count     uint64 `json:"count,omitempty"`
-	Error     string `json:"error,omitempty"`
-}
-
-func Assign(server string, count uint64, replication string, collection string, ttl string) (*AssignResult, error) {
-	values := make(url.Values)
-	values.Add("count", strconv.FormatUint(count, 10))
-	if replication != "" {
-		values.Add("replication", replication)
-	}
-	if collection != "" {
-		values.Add("collection", collection)
-	}
-	if ttl != "" {
-		values.Add("ttl", ttl)
-	}
-	jsonBlob, err := util.Post("http://"+server+"/dir/assign", values)
-	glog.V(2).Info("assign result :", string(jsonBlob))
-	if err != nil {
-		return nil, err
-	}
-	var ret AssignResult
-	err = json.Unmarshal(jsonBlob, &ret)
-	if err != nil {
-		return nil, fmt.Errorf("/dir/assign result JSON unmarshal error:%v, json:%s", err, string(jsonBlob))
-	}
-	if ret.Count <= 0 {
-		return nil, errors.New(ret.Error)
-	}
-	return &ret, nil
-}

go/operation/delete_content.go

@@ -1,104 +0,0 @@
-package operation
-
-import (
-	"encoding/json"
-	"errors"
-	"net/url"
-	"strings"
-	"sync"
-
-	"github.com/chrislusf/weed-fs/go/security"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-type DeleteResult struct {
-	Fid   string `json:"fid"`
-	Size  int    `json:"size"`
-	Error string `json:"error,omitempty"`
-}
-
-func DeleteFile(master string, fileId string, jwt security.EncodedJwt) error {
-	fileUrl, err := LookupFileId(master, fileId)
-	if err != nil {
-		return err
-	}
-	return util.Delete(fileUrl, jwt)
-}
-
-func ParseFileId(fid string) (vid string, key_cookie string, err error) {
-	commaIndex := strings.Index(fid, ",")
-	if commaIndex <= 0 {
-		return "", "", errors.New("Wrong fid format.")
-	}
-	return fid[:commaIndex], fid[commaIndex+1:], nil
-}
-
-type DeleteFilesResult struct {
-	Errors  []string
-	Results []DeleteResult
-}
-
-func DeleteFiles(master string, fileIds []string) (*DeleteFilesResult, error) {
-	vid_to_fileIds := make(map[string][]string)
-	ret := &DeleteFilesResult{}
-	var vids []string
-	for _, fileId := range fileIds {
-		vid, _, err := ParseFileId(fileId)
-		if err != nil {
-			ret.Results = append(ret.Results, DeleteResult{Fid: vid, Error: err.Error()})
-			continue
-		}
-		if _, ok := vid_to_fileIds[vid]; !ok {
-			vid_to_fileIds[vid] = make([]string, 0)
-			vids = append(vids, vid)
-		}
-		vid_to_fileIds[vid] = append(vid_to_fileIds[vid], fileId)
-	}
-
-	lookupResults, err := LookupVolumeIds(master, vids)
-	if err != nil {
-		return ret, err
-	}
-
-	server_to_fileIds := make(map[string][]string)
-	for vid, result := range lookupResults {
-		if result.Error != "" {
-			ret.Errors = append(ret.Errors, result.Error)
-			continue
-		}
-		for _, location := range result.Locations {
-			if _, ok := server_to_fileIds[location.Url]; !ok {
-				server_to_fileIds[location.Url] = make([]string, 0)
-			}
-			server_to_fileIds[location.Url] = append(
-				server_to_fileIds[location.Url], vid_to_fileIds[vid]...)
-		}
-	}
-
-	var wg sync.WaitGroup
-	for server, fidList := range server_to_fileIds {
-		wg.Add(1)
-		go func(server string, fidList []string) {
-			defer wg.Done()
-			values := make(url.Values)
-			for _, fid := range fidList {
-				values.Add("fid", fid)
-			}
-			jsonBlob, err := util.Post("http://"+server+"/delete", values)
-			if err != nil {
-				ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
-				return
-			}
-			var result []DeleteResult
-			err = json.Unmarshal(jsonBlob, &result)
-			if err != nil {
-				ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
-				return
-			}
-			ret.Results = append(ret.Results, result...)
-		}(server, fidList)
-	}
-	wg.Wait()
-
-	return ret, nil
-}

go/operation/list_masters.go

@@ -1,32 +0,0 @@
-package operation
-
-import (
-	"encoding/json"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-type ClusterStatusResult struct {
-	IsLeader bool     `json:"IsLeader,omitempty"`
-	Leader   string   `json:"Leader,omitempty"`
-	Peers    []string `json:"Peers,omitempty"`
-}
-
-func ListMasters(server string) ([]string, error) {
-	jsonBlob, err := util.Get("http://" + server + "/cluster/status")
-	glog.V(2).Info("list masters result :", string(jsonBlob))
-	if err != nil {
-		return nil, err
-	}
-	var ret ClusterStatusResult
-	err = json.Unmarshal(jsonBlob, &ret)
-	if err != nil {
-		return nil, err
-	}
-	masters := ret.Peers
-	if ret.IsLeader {
-		masters = append(masters, ret.Leader)
-	}
-	return masters, nil
-}

go/operation/lookup_vid_cache.go

@@ -1,41 +0,0 @@
-package operation
-
-import (
-	"errors"
-	"strconv"
-	"time"
-)
-
-type VidInfo struct {
-	Locations       []Location
-	NextRefreshTime time.Time
-}
-type VidCache struct {
-	cache []VidInfo
-}
-
-func (vc *VidCache) Get(vid string) ([]Location, error) {
-	id, _ := strconv.Atoi(vid)
-	if 0 < id && id <= len(vc.cache) {
-		if vc.cache[id-1].Locations == nil {
-			return nil, errors.New("Not Set")
-		}
-		if vc.cache[id-1].NextRefreshTime.Before(time.Now()) {
-			return nil, errors.New("Expired")
-		}
-		return vc.cache[id-1].Locations, nil
-	}
-	return nil, errors.New("Not Found")
-}
-func (vc *VidCache) Set(vid string, locations []Location, duration time.Duration) {
-	id, _ := strconv.Atoi(vid)
-	if id >= len(vc.cache) {
-		for i := id - len(vc.cache); i > 0; i-- {
-			vc.cache = append(vc.cache, VidInfo{})
-		}
-	}
-
-	vc.cache[id-1].Locations = locations
-	vc.cache[id-1].NextRefreshTime = time.Now().Add(duration)
-
-}

go/operation/submit.go

@@ -1,167 +0,0 @@
-package operation
-
-import (
-	"bytes"
-	"io"
-	"mime"
-	"os"
-	"path"
-	"strconv"
-	"strings"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/security"
-)
-
-type FilePart struct {
-	Reader      io.Reader
-	FileName    string
-	FileSize    int64
-	IsGzipped   bool
-	MimeType    string
-	ModTime     int64 //in seconds
-	Replication string
-	Collection  string
-	Ttl         string
-	Server      string //this comes from assign result
-	Fid         string //this comes from assign result, but customizable
-}
-
-type SubmitResult struct {
-	FileName string `json:"fileName,omitempty"`
-	FileUrl  string `json:"fileUrl,omitempty"`
-	Fid      string `json:"fid,omitempty"`
-	Size     uint32 `json:"size,omitempty"`
-	Error    string `json:"error,omitempty"`
-}
-
-func SubmitFiles(master string, files []FilePart,
-	replication string, collection string, ttl string, maxMB int,
-	secret security.Secret,
-) ([]SubmitResult, error) {
-	results := make([]SubmitResult, len(files))
-	for index, file := range files {
-		results[index].FileName = file.FileName
-	}
-	ret, err := Assign(master, uint64(len(files)), replication, collection, ttl)
-	if err != nil {
-		for index, _ := range files {
-			results[index].Error = err.Error()
-		}
-		return results, err
-	}
-	for index, file := range files {
-		file.Fid = ret.Fid
-		if index > 0 {
-			file.Fid = file.Fid + "_" + strconv.Itoa(index)
-		}
-		file.Server = ret.Url
-		file.Replication = replication
-		file.Collection = collection
-		results[index].Size, err = file.Upload(maxMB, master, secret)
-		if err != nil {
-			results[index].Error = err.Error()
-		}
-		results[index].Fid = file.Fid
-		results[index].FileUrl = file.Server + "/" + file.Fid
-	}
-	return results, nil
-}
-
-func NewFileParts(fullPathFilenames []string) (ret []FilePart, err error) {
-	ret = make([]FilePart, len(fullPathFilenames))
-	for index, file := range fullPathFilenames {
-		if ret[index], err = newFilePart(file); err != nil {
-			return
-		}
-	}
-	return
-}
-func newFilePart(fullPathFilename string) (ret FilePart, err error) {
-	fh, openErr := os.Open(fullPathFilename)
-	if openErr != nil {
-		glog.V(0).Info("Failed to open file: ", fullPathFilename)
-		return ret, openErr
-	}
-	ret.Reader = fh
-
-	if fi, fiErr := fh.Stat(); fiErr != nil {
-		glog.V(0).Info("Failed to stat file:", fullPathFilename)
-		return ret, fiErr
-	} else {
-		ret.ModTime = fi.ModTime().UTC().Unix()
-		ret.FileSize = fi.Size()
-	}
-	ext := strings.ToLower(path.Ext(fullPathFilename))
-	ret.IsGzipped = ext == ".gz"
-	if ret.IsGzipped {
-		ret.FileName = fullPathFilename[0 : len(fullPathFilename)-3]
-	}
-	ret.FileName = fullPathFilename
-	if ext != "" {
-		ret.MimeType = mime.TypeByExtension(ext)
-	}
-
-	return ret, nil
-}
-
-func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (retSize uint32, err error) {
-	jwt := security.GenJwt(secret, fi.Fid)
-	fileUrl := "http://" + fi.Server + "/" + fi.Fid
-	if fi.ModTime != 0 {
-		fileUrl += "?ts=" + strconv.Itoa(int(fi.ModTime))
-	}
-	if closer, ok := fi.Reader.(io.Closer); ok {
-		defer closer.Close()
-	}
-	if maxMB > 0 && fi.FileSize > int64(maxMB*1024*1024) {
-		chunkSize := int64(maxMB * 1024 * 1024)
-		chunks := fi.FileSize/chunkSize + 1
-		var fids []string
-		for i := int64(0); i < chunks; i++ {
-			id, count, e := upload_one_chunk(
-				fi.FileName+"-"+strconv.FormatInt(i+1, 10),
-				io.LimitReader(fi.Reader, chunkSize),
-				master, fi.Replication, fi.Collection, fi.Ttl,
-				jwt)
-			if e != nil {
-				return 0, e
-			}
-			fids = append(fids, id)
-			retSize += count
-		}
-		err = upload_file_id_list(fileUrl, fi.FileName+"-list", fids, jwt)
-	} else {
-		ret, e := Upload(fileUrl, fi.FileName, fi.Reader, fi.IsGzipped, fi.MimeType, jwt)
-		if e != nil {
-			return 0, e
-		}
-		return ret.Size, e
-	}
-	return
-}
-
-func upload_one_chunk(filename string, reader io.Reader, master,
-	replication string, collection string, ttl string, jwt security.EncodedJwt,
-) (fid string, size uint32, e error) {
-	ret, err := Assign(master, 1, replication, collection, ttl)
-	if err != nil {
-		return "", 0, err
-	}
-	fileUrl, fid := "http://"+ret.Url+"/"+ret.Fid, ret.Fid
-	glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
-	uploadResult, uploadError := Upload(fileUrl, filename, reader, false,
-		"application/octet-stream", jwt)
-	if uploadError != nil {
-		return fid, 0, uploadError
-	}
-	return fid, uploadResult.Size, nil
-}
-
-func upload_file_id_list(fileUrl, filename string, fids []string, jwt security.EncodedJwt) error {
-	var buf bytes.Buffer
-	buf.WriteString(strings.Join(fids, "\n"))
-	glog.V(4).Info("Uploading final list ", filename, " to ", fileUrl, "...")
-	_, e := Upload(fileUrl, filename, &buf, false, "text/plain", jwt)
-	return e
-}

go/operation/system_message.pb.go

@@ -1,203 +0,0 @@
-// Code generated by protoc-gen-go.
-// source: system_message.proto
-// DO NOT EDIT!
-
-/*
-Package operation is a generated protocol buffer package.
-
-It is generated from these files:
-	system_message.proto
-
-It has these top-level messages:
-	VolumeInformationMessage
-	JoinMessage
-*/
-package operation
-
-import proto "github.com/golang/protobuf/proto"
-import math "math"
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = math.Inf
-
-type VolumeInformationMessage struct {
-	Id               *uint32 `protobuf:"varint,1,req,name=id" json:"id,omitempty"`
-	Size             *uint64 `protobuf:"varint,2,req,name=size" json:"size,omitempty"`
-	Collection       *string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"`
-	FileCount        *uint64 `protobuf:"varint,4,req,name=file_count" json:"file_count,omitempty"`
-	DeleteCount      *uint64 `protobuf:"varint,5,req,name=delete_count" json:"delete_count,omitempty"`
-	DeletedByteCount *uint64 `protobuf:"varint,6,req,name=deleted_byte_count" json:"deleted_byte_count,omitempty"`
-	ReadOnly         *bool   `protobuf:"varint,7,opt,name=read_only" json:"read_only,omitempty"`
-	ReplicaPlacement *uint32 `protobuf:"varint,8,req,name=replica_placement" json:"replica_placement,omitempty"`
-	Version          *uint32 `protobuf:"varint,9,opt,name=version,def=2" json:"version,omitempty"`
-	Ttl              *uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"`
-	XXX_unrecognized []byte  `json:"-"`
-}
-
-func (m *VolumeInformationMessage) Reset()         { *m = VolumeInformationMessage{} }
-func (m *VolumeInformationMessage) String() string { return proto.CompactTextString(m) }
-func (*VolumeInformationMessage) ProtoMessage()    {}
-
-const Default_VolumeInformationMessage_Version uint32 = 2
-
-func (m *VolumeInformationMessage) GetId() uint32 {
-	if m != nil && m.Id != nil {
-		return *m.Id
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetSize() uint64 {
-	if m != nil && m.Size != nil {
-		return *m.Size
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetCollection() string {
-	if m != nil && m.Collection != nil {
-		return *m.Collection
-	}
-	return ""
-}
-
-func (m *VolumeInformationMessage) GetFileCount() uint64 {
-	if m != nil && m.FileCount != nil {
-		return *m.FileCount
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetDeleteCount() uint64 {
-	if m != nil && m.DeleteCount != nil {
-		return *m.DeleteCount
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetDeletedByteCount() uint64 {
-	if m != nil && m.DeletedByteCount != nil {
-		return *m.DeletedByteCount
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetReadOnly() bool {
-	if m != nil && m.ReadOnly != nil {
-		return *m.ReadOnly
-	}
-	return false
-}
-
-func (m *VolumeInformationMessage) GetReplicaPlacement() uint32 {
-	if m != nil && m.ReplicaPlacement != nil {
-		return *m.ReplicaPlacement
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetVersion() uint32 {
-	if m != nil && m.Version != nil {
-		return *m.Version
-	}
-	return Default_VolumeInformationMessage_Version
-}
-
-func (m *VolumeInformationMessage) GetTtl() uint32 {
-	if m != nil && m.Ttl != nil {
-		return *m.Ttl
-	}
-	return 0
-}
-
-type JoinMessage struct {
-	IsInit           *bool                       `protobuf:"varint,1,opt,name=is_init" json:"is_init,omitempty"`
-	Ip               *string                     `protobuf:"bytes,2,req,name=ip" json:"ip,omitempty"`
-	Port             *uint32                     `protobuf:"varint,3,req,name=port" json:"port,omitempty"`
-	PublicUrl        *string                     `protobuf:"bytes,4,opt,name=public_url" json:"public_url,omitempty"`
-	MaxVolumeCount   *uint32                     `protobuf:"varint,5,req,name=max_volume_count" json:"max_volume_count,omitempty"`
-	MaxFileKey       *uint64                     `protobuf:"varint,6,req,name=max_file_key" json:"max_file_key,omitempty"`
-	DataCenter       *string                     `protobuf:"bytes,7,opt,name=data_center" json:"data_center,omitempty"`
-	Rack             *string                     `protobuf:"bytes,8,opt,name=rack" json:"rack,omitempty"`
-	Volumes          []*VolumeInformationMessage `protobuf:"bytes,9,rep,name=volumes" json:"volumes,omitempty"`
-	AdminPort        *uint32                     `protobuf:"varint,10,opt,name=admin_port" json:"admin_port,omitempty"`
-	XXX_unrecognized []byte                      `json:"-"`
-}
-
-func (m *JoinMessage) Reset()         { *m = JoinMessage{} }
-func (m *JoinMessage) String() string { return proto.CompactTextString(m) }
-func (*JoinMessage) ProtoMessage()    {}
-
-func (m *JoinMessage) GetIsInit() bool {
-	if m != nil && m.IsInit != nil {
-		return *m.IsInit
-	}
-	return false
-}
-
-func (m *JoinMessage) GetIp() string {
-	if m != nil && m.Ip != nil {
-		return *m.Ip
-	}
-	return ""
-}
-
-func (m *JoinMessage) GetPort() uint32 {
-	if m != nil && m.Port != nil {
-		return *m.Port
-	}
-	return 0
-}
-
-func (m *JoinMessage) GetPublicUrl() string {
-	if m != nil && m.PublicUrl != nil {
-		return *m.PublicUrl
-	}
-	return ""
-}
-
-func (m *JoinMessage) GetMaxVolumeCount() uint32 {
-	if m != nil && m.MaxVolumeCount != nil {
-		return *m.MaxVolumeCount
-	}
-	return 0
-}
-
-func (m *JoinMessage) GetMaxFileKey() uint64 {
-	if m != nil && m.MaxFileKey != nil {
-		return *m.MaxFileKey
-	}
-	return 0
-}
-
-func (m *JoinMessage) GetDataCenter() string {
-	if m != nil && m.DataCenter != nil {
-		return *m.DataCenter
-	}
-	return ""
-}
-
-func (m *JoinMessage) GetRack() string {
-	if m != nil && m.Rack != nil {
-		return *m.Rack
-	}
-	return ""
-}
-
-func (m *JoinMessage) GetVolumes() []*VolumeInformationMessage {
-	if m != nil {
-		return m.Volumes
-	}
-	return nil
-}
-
-func (m *JoinMessage) GetAdminPort() uint32 {
-	if m != nil && m.AdminPort != nil {
-		return *m.AdminPort
-	}
-	return 0
-}
-
-func init() {
-}

go/operation/system_message_test.go

@@ -1,59 +0,0 @@
-package operation
-
-import (
-	"encoding/json"
-	"log"
-	"testing"
-
-	"github.com/golang/protobuf/proto"
-)
-
-func TestSerialDeserial(t *testing.T) {
-	volumeMessage := &VolumeInformationMessage{
-		Id:               proto.Uint32(12),
-		Size:             proto.Uint64(2341234),
-		Collection:       proto.String("benchmark"),
-		FileCount:        proto.Uint64(2341234),
-		DeleteCount:      proto.Uint64(234),
-		DeletedByteCount: proto.Uint64(21234),
-		ReadOnly:         proto.Bool(false),
-		ReplicaPlacement: proto.Uint32(210),
-		Version:          proto.Uint32(2),
-	}
-	var volumeMessages []*VolumeInformationMessage
-	volumeMessages = append(volumeMessages, volumeMessage)
-
-	joinMessage := &JoinMessage{
-		IsInit:         proto.Bool(true),
-		Ip:             proto.String("127.0.3.12"),
-		Port:           proto.Uint32(34546),
-		PublicUrl:      proto.String("localhost:2342"),
-		MaxVolumeCount: proto.Uint32(210),
-		MaxFileKey:     proto.Uint64(324234423),
-		DataCenter:     proto.String("dc1"),
-		Rack:           proto.String("rack2"),
-		Volumes:        volumeMessages,
-	}
-
-	data, err := proto.Marshal(joinMessage)
-	if err != nil {
-		log.Fatal("marshaling error: ", err)
-	}
-	newMessage := &JoinMessage{}
-	err = proto.Unmarshal(data, newMessage)
-	if err != nil {
-		log.Fatal("unmarshaling error: ", err)
-	}
-	log.Println("The pb data size is", len(data))
-
-	jsonData, jsonError := json.Marshal(joinMessage)
-	if jsonError != nil {
-		log.Fatal("json marshaling error: ", jsonError)
-	}
-	log.Println("The json data size is", len(jsonData), string(jsonData))
-
-	// Now test and newTest contain the same data.
-	if *joinMessage.PublicUrl != *newMessage.PublicUrl {
-		log.Fatalf("data mismatch %q != %q", *joinMessage.PublicUrl, *newMessage.PublicUrl)
-	}
-}

go/operation/upload_content.go

@@ -1,96 +0,0 @@
-package operation
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"mime"
-	"mime/multipart"
-	"net/http"
-	"net/textproto"
-	"path/filepath"
-	"strings"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/security"
-)
-
-type UploadResult struct {
-	Name  string `json:"name,omitempty"`
-	Size  uint32 `json:"size,omitempty"`
-	Error string `json:"error,omitempty"`
-}
-
-var (
-	client *http.Client
-)
-
-func init() {
-	client = &http.Client{Transport: &http.Transport{
-		MaxIdleConnsPerHost: 1024,
-	}}
-}
-
-var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
-
-func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, jwt security.EncodedJwt) (*UploadResult, error) {
-	return upload_content(uploadUrl, func(w io.Writer) (err error) {
-		_, err = io.Copy(w, reader)
-		return
-	}, filename, isGzipped, mtype, jwt)
-}
-func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, jwt security.EncodedJwt) (*UploadResult, error) {
-	body_buf := bytes.NewBufferString("")
-	body_writer := multipart.NewWriter(body_buf)
-	h := make(textproto.MIMEHeader)
-	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
-	if mtype == "" {
-		mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
-	}
-	if mtype != "" {
-		h.Set("Content-Type", mtype)
-	}
-	if isGzipped {
-		h.Set("Content-Encoding", "gzip")
-	}
-	if jwt != "" {
-		h.Set("Authorization", "BEARER "+string(jwt))
-	}
-	file_writer, cp_err := body_writer.CreatePart(h)
-	if cp_err != nil {
-		glog.V(0).Infoln("error creating form file", cp_err.Error())
-		return nil, cp_err
-	}
-	if err := fillBufferFunction(file_writer); err != nil {
-		glog.V(0).Infoln("error copying data", err)
-		return nil, err
-	}
-	content_type := body_writer.FormDataContentType()
-	if err := body_writer.Close(); err != nil {
-		glog.V(0).Infoln("error closing body", err)
-		return nil, err
-	}
-	resp, post_err := client.Post(uploadUrl, content_type, body_buf)
-	if post_err != nil {
-		glog.V(0).Infoln("failing to upload to", uploadUrl, post_err.Error())
-		return nil, post_err
-	}
-	defer resp.Body.Close()
-	resp_body, ra_err := ioutil.ReadAll(resp.Body)
-	if ra_err != nil {
-		return nil, ra_err
-	}
-	var ret UploadResult
-	unmarshal_err := json.Unmarshal(resp_body, &ret)
-	if unmarshal_err != nil {
-		glog.V(0).Infoln("failing to read upload resonse", uploadUrl, string(resp_body))
-		return nil, unmarshal_err
-	}
-	if ret.Error != "" {
-		return nil, errors.New(ret.Error)
-	}
-	return &ret, nil
-}

go/proto/Makefile

@@ -1,4 +0,0 @@
-TARG=../operation
-
-all:
-	protoc --go_out=$(TARG) system_message.proto

go/proto/system_message.proto

@@ -1,27 +0,0 @@
-package operation;
-
-message VolumeInformationMessage {
-  required uint32 id = 1;
-  required uint64 size = 2;
-  optional string collection = 3;
-  required uint64 file_count = 4;
-  required uint64 delete_count = 5;
-  required uint64 deleted_byte_count = 6;
-  optional bool read_only = 7;
-  required uint32 replica_placement = 8;
-  optional uint32 version = 9 [default=2];
-  optional uint32 ttl = 10;
-}
-
-message JoinMessage {
-  optional bool   is_init = 1;
-  required string ip = 2;
-  required uint32 port = 3;
-  optional string public_url = 4;
-  required uint32 max_volume_count = 5;
-  required uint64 max_file_key = 6;
-  optional string data_center = 7;
-  optional string rack = 8;
-  repeated VolumeInformationMessage volumes = 9;
-  optional uint32 admin_port = 10;
-}

go/security/guard.go

@@ -1,129 +0,0 @@
-package security
-
-import (
-	"errors"
-	"fmt"
-	"net"
-	"net/http"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-)
-
-var (
-	ErrUnauthorized = errors.New("unauthorized token")
-)
-
-/*
-Guard is to ensure data access security.
-There are 2 ways to check access:
-1. white list. It's checking request ip address.
-2. JSON Web Token(JWT) generated from secretKey.
-  The jwt can come from:
-  1. url parameter jwt=...
-  2. request header "Authorization"
-  3. cookie with the name "jwt"
-
-The white list is checked first because it is easy.
-Then the JWT is checked.
-
-The Guard will also check these claims if provided:
-1. "exp" Expiration Time
-2. "nbf" Not Before
-
-Generating JWT:
-1. use HS256 to sign
-2. optionally set "exp", "nbf" fields, in Unix time,
-   the number of seconds elapsed since January 1, 1970 UTC.
-
-Referenced:
-https://github.com/pkieltyka/jwtauth/blob/master/jwtauth.go
-
-*/
-type Guard struct {
-	whiteList []string
-	SecretKey Secret
-
-	isActive bool
-}
-
-func NewGuard(whiteList []string, secretKey string) *Guard {
-	g := &Guard{whiteList: whiteList, SecretKey: Secret(secretKey)}
-	g.isActive = len(g.whiteList) != 0 || len(g.SecretKey) != 0
-	return g
-}
-
-func (g *Guard) WhiteList(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
-	if !g.isActive {
-		//if no security needed, just skip all checkings
-		return f
-	}
-	return func(w http.ResponseWriter, r *http.Request) {
-		if err := g.checkWhiteList(w, r); err != nil {
-			w.WriteHeader(http.StatusUnauthorized)
-			return
-		}
-		f(w, r)
-	}
-}
-
-func (g *Guard) Secure(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
-	if !g.isActive {
-		//if no security needed, just skip all checkings
-		return f
-	}
-	return func(w http.ResponseWriter, r *http.Request) {
-		if err := g.checkJwt(w, r); err != nil {
-			w.WriteHeader(http.StatusUnauthorized)
-			return
-		}
-		f(w, r)
-	}
-}
-
-func (g *Guard) checkWhiteList(w http.ResponseWriter, r *http.Request) error {
-	if len(g.whiteList) == 0 {
-		return nil
-	}
-
-	host, _, err := net.SplitHostPort(r.RemoteAddr)
-	if err == nil {
-		for _, ip := range g.whiteList {
-			if ip == host {
-				return nil
-			}
-		}
-	}
-
-	glog.V(1).Infof("Not in whitelist: %s", r.RemoteAddr)
-	return fmt.Errorf("Not in whitelis: %s", r.RemoteAddr)
-}
-
-func (g *Guard) checkJwt(w http.ResponseWriter, r *http.Request) error {
-	if g.checkWhiteList(w, r) == nil {
-		return nil
-	}
-
-	if len(g.SecretKey) == 0 {
-		return nil
-	}
-
-	tokenStr := GetJwt(r)
-
-	if tokenStr == "" {
-		return ErrUnauthorized
-	}
-
-	// Verify the token
-	token, err := DecodeJwt(g.SecretKey, tokenStr)
-	if err != nil {
-		glog.V(1).Infof("Token verification error from %s: %v", r.RemoteAddr, err)
-		return ErrUnauthorized
-	}
-	if !token.Valid {
-		glog.V(1).Infof("Token invliad from %s: %v", r.RemoteAddr, tokenStr)
-		return ErrUnauthorized
-	}
-
-	glog.V(1).Infof("No permission from %s", r.RemoteAddr)
-	return fmt.Errorf("No write permisson from %s", r.RemoteAddr)
-}

go/security/jwt.go

@@ -1,72 +0,0 @@
-package security
-
-import (
-	"net/http"
-	"strings"
-
-	"time"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	jwt "github.com/dgrijalva/jwt-go"
-)
-
-type EncodedJwt string
-type Secret string
-
-func GenJwt(secret Secret, fileId string) EncodedJwt {
-	if secret == "" {
-		return ""
-	}
-
-	t := jwt.New(jwt.GetSigningMethod("HS256"))
-	t.Claims["exp"] = time.Now().Unix() + 10
-	t.Claims["sub"] = fileId
-	encoded, e := t.SignedString(secret)
-	if e != nil {
-		glog.V(0).Infof("Failed to sign claims: %v", t.Claims)
-		return ""
-	}
-	return EncodedJwt(encoded)
-}
-
-func GetJwt(r *http.Request) EncodedJwt {
-
-	// Get token from query params
-	tokenStr := r.URL.Query().Get("jwt")
-
-	// Get token from authorization header
-	if tokenStr == "" {
-		bearer := r.Header.Get("Authorization")
-		if len(bearer) > 7 && strings.ToUpper(bearer[0:6]) == "BEARER" {
-			tokenStr = bearer[7:]
-		}
-	}
-
-	// Get token from cookie
-	if tokenStr == "" {
-		cookie, err := r.Cookie("jwt")
-		if err == nil {
-			tokenStr = cookie.Value
-		}
-	}
-
-	return EncodedJwt(tokenStr)
-}
-
-func EncodeJwt(secret Secret, claims map[string]interface{}) (EncodedJwt, error) {
-	if secret == "" {
-		return "", nil
-	}
-
-	t := jwt.New(jwt.GetSigningMethod("HS256"))
-	t.Claims = claims
-	encoded, e := t.SignedString(secret)
-	return EncodedJwt(encoded), e
-}
-
-func DecodeJwt(secret Secret, tokenString EncodedJwt) (token *jwt.Token, err error) {
-	// check exp, nbf
-	return jwt.Parse(string(tokenString), func(token *jwt.Token) (interface{}, error) {
-		return secret, nil
-	})
-}

go/stats/disk.go

@@ -1,14 +0,0 @@
-package stats
-
-type DiskStatus struct {
-	Dir  string
-	All  uint64
-	Used uint64
-	Free uint64
-}
-
-func NewDiskStatus(path string) (disk *DiskStatus) {
-	disk = &DiskStatus{Dir: path}
-	disk.fillInStatus()
-	return
-}

go/stats/disk_notsupported.go

@@ -1,7 +0,0 @@
-// +build windows openbsd netbsd plan9 solaris
-
-package stats
-
-func (disk *DiskStatus) fillInStatus() {
-	return
-}

go/stats/memory.go

@@ -1,28 +0,0 @@
-package stats
-
-import (
-	"runtime"
-)
-
-type MemStatus struct {
-	Goroutines int
-	All        uint64
-	Used       uint64
-	Free       uint64
-	Self       uint64
-	Heap       uint64
-	Stack      uint64
-}
-
-func MemStat() MemStatus {
-	mem := MemStatus{}
-	mem.Goroutines = runtime.NumGoroutine()
-	memStat := new(runtime.MemStats)
-	runtime.ReadMemStats(memStat)
-	mem.Self = memStat.Alloc
-	mem.Heap = memStat.HeapAlloc
-	mem.Stack = memStat.StackInuse
-
-	mem.fillInStatus()
-	return mem
-}

go/stats/memory_notsupported.go

@@ -1,7 +0,0 @@
-// +build !linux
-
-package stats
-
-func (mem *MemStatus) fillInStatus() {
-	return
-}

go/storage/compact_map.go

@@ -1,188 +0,0 @@
-package storage
-
-type NeedleValue struct {
-	Key    Key
-	Offset uint32 `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
-	Size   uint32 `comment:"Size of the data portion"`
-}
-
-const (
-	batch = 100000
-)
-
-type Key uint64
-
-type CompactSection struct {
-	values   []NeedleValue
-	overflow map[Key]NeedleValue
-	start    Key
-	end      Key
-	counter  int
-}
-
-func NewCompactSection(start Key) *CompactSection {
-	return &CompactSection{
-		values:   make([]NeedleValue, batch),
-		overflow: make(map[Key]NeedleValue),
-		start:    start,
-	}
-}
-
-//return old entry size
-func (cs *CompactSection) Set(key Key, offset uint32, size uint32) uint32 {
-	ret := uint32(0)
-	if key > cs.end {
-		cs.end = key
-	}
-	if i := cs.binarySearchValues(key); i >= 0 {
-		ret = cs.values[i].Size
-		//println("key", key, "old size", ret)
-		cs.values[i].Offset, cs.values[i].Size = offset, size
-	} else {
-		needOverflow := cs.counter >= batch
-		needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > key
-		if needOverflow {
-			//println("start", cs.start, "counter", cs.counter, "key", key)
-			if oldValue, found := cs.overflow[key]; found {
-				ret = oldValue.Size
-			}
-			cs.overflow[key] = NeedleValue{Key: key, Offset: offset, Size: size}
-		} else {
-			p := &cs.values[cs.counter]
-			p.Key, p.Offset, p.Size = key, offset, size
-			//println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
-			cs.counter++
-		}
-	}
-	return ret
-}
-
-//return old entry size
-func (cs *CompactSection) Delete(key Key) uint32 {
-	ret := uint32(0)
-	if i := cs.binarySearchValues(key); i >= 0 {
-		if cs.values[i].Size > 0 {
-			ret = cs.values[i].Size
-			cs.values[i].Size = 0
-		}
-	}
-	if v, found := cs.overflow[key]; found {
-		delete(cs.overflow, key)
-		ret = v.Size
-	}
-	return ret
-}
-func (cs *CompactSection) Get(key Key) (*NeedleValue, bool) {
-	if v, ok := cs.overflow[key]; ok {
-		return &v, true
-	}
-	if i := cs.binarySearchValues(key); i >= 0 {
-		return &cs.values[i], true
-	}
-	return nil, false
-}
-func (cs *CompactSection) binarySearchValues(key Key) int {
-	l, h := 0, cs.counter-1
-	if h >= 0 && cs.values[h].Key < key {
-		return -2
-	}
-	//println("looking for key", key)
-	for l <= h {
-		m := (l + h) / 2
-		//println("mid", m, "key", cs.values[m].Key, cs.values[m].Offset, cs.values[m].Size)
-		if cs.values[m].Key < key {
-			l = m + 1
-		} else if key < cs.values[m].Key {
-			h = m - 1
-		} else {
-			//println("found", m)
-			return m
-		}
-	}
-	return -1
-}
-
-//This map assumes mostly inserting increasing keys
-type CompactMap struct {
-	list []*CompactSection
-}
-
-func NewCompactMap() CompactMap {
-	return CompactMap{}
-}
-
-func (cm *CompactMap) Set(key Key, offset uint32, size uint32) uint32 {
-	x := cm.binarySearchCompactSection(key)
-	if x < 0 {
-		//println(x, "creating", len(cm.list), "section, starting", key)
-		cm.list = append(cm.list, NewCompactSection(key))
-		x = len(cm.list) - 1
-		//keep compact section sorted by start
-		for x > 0 {
-			if cm.list[x-1].start > cm.list[x].start {
-				cm.list[x-1], cm.list[x] = cm.list[x], cm.list[x-1]
-				x = x - 1
-			} else {
-				break
-			}
-		}
-	}
-	return cm.list[x].Set(key, offset, size)
-}
-func (cm *CompactMap) Delete(key Key) uint32 {
-	x := cm.binarySearchCompactSection(key)
-	if x < 0 {
-		return uint32(0)
-	}
-	return cm.list[x].Delete(key)
-}
-func (cm *CompactMap) Get(key Key) (*NeedleValue, bool) {
-	x := cm.binarySearchCompactSection(key)
-	if x < 0 {
-		return nil, false
-	}
-	return cm.list[x].Get(key)
-}
-func (cm *CompactMap) binarySearchCompactSection(key Key) int {
-	l, h := 0, len(cm.list)-1
-	if h < 0 {
-		return -5
-	}
-	if cm.list[h].start <= key {
-		if cm.list[h].counter < batch || key <= cm.list[h].end {
-			return h
-		}
-		return -4
-	}
-	for l <= h {
-		m := (l + h) / 2
-		if key < cm.list[m].start {
-			h = m - 1
-		} else { // cm.list[m].start <= key
-			if cm.list[m+1].start <= key {
-				l = m + 1
-			} else {
-				return m
-			}
-		}
-	}
-	return -3
-}
-
-func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
-	for _, cs := range cm.list {
-		for _, v := range cs.overflow {
-			if err := visit(v); err != nil {
-				return err
-			}
-		}
-		for _, v := range cs.values {
-			if _, found := cs.overflow[v.Key]; !found {
-				if err := visit(v); err != nil {
-					return err
-				}
-			}
-		}
-	}
-	return nil
-}

go/storage/compact_map_perf_test.go

@@ -1,45 +0,0 @@
-package storage
-
-import (
-	"log"
-	"os"
-	"testing"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-func TestMemoryUsage(t *testing.T) {
-
-	indexFile, ie := os.OpenFile("../../test/sample.idx", os.O_RDWR|os.O_RDONLY, 0644)
-	if ie != nil {
-		log.Fatalln(ie)
-	}
-	LoadNewNeedleMap(indexFile)
-
-}
-
-func LoadNewNeedleMap(file *os.File) CompactMap {
-	m := NewCompactMap()
-	bytes := make([]byte, 16*1024)
-	count, e := file.Read(bytes)
-	if count > 0 {
-		fstat, _ := file.Stat()
-		glog.V(0).Infoln("Loading index file", fstat.Name(), "size", fstat.Size())
-	}
-	for count > 0 && e == nil {
-		for i := 0; i < count; i += 16 {
-			key := util.BytesToUint64(bytes[i : i+8])
-			offset := util.BytesToUint32(bytes[i+8 : i+12])
-			size := util.BytesToUint32(bytes[i+12 : i+16])
-			if offset > 0 {
-				m.Set(Key(key), offset, size)
-			} else {
-				//delete(m, key)
-			}
-		}
-
-		count, e = file.Read(bytes)
-	}
-	return m
-}

go/storage/compact_map_test.go

@@ -1,77 +0,0 @@
-package storage
-
-import (
-	"testing"
-)
-
-func TestIssue52(t *testing.T) {
-	m := NewCompactMap()
-	m.Set(Key(10002), 10002, 10002)
-	if element, ok := m.Get(Key(10002)); ok {
-		println("key", 10002, "ok", ok, element.Key, element.Offset, element.Size)
-	}
-	m.Set(Key(10001), 10001, 10001)
-	if element, ok := m.Get(Key(10002)); ok {
-		println("key", 10002, "ok", ok, element.Key, element.Offset, element.Size)
-	} else {
-		t.Fatal("key 10002 missing after setting 10001")
-	}
-}
-
-func TestXYZ(t *testing.T) {
-	m := NewCompactMap()
-	for i := uint32(0); i < 100*batch; i += 2 {
-		m.Set(Key(i), i, i)
-	}
-
-	for i := uint32(0); i < 100*batch; i += 37 {
-		m.Delete(Key(i))
-	}
-
-	for i := uint32(0); i < 10*batch; i += 3 {
-		m.Set(Key(i), i+11, i+5)
-	}
-
-	//	for i := uint32(0); i < 100; i++ {
-	//		if v := m.Get(Key(i)); v != nil {
-	//			glog.V(4).Infoln(i, "=", v.Key, v.Offset, v.Size)
-	//		}
-	//	}
-
-	for i := uint32(0); i < 10*batch; i++ {
-		v, ok := m.Get(Key(i))
-		if i%3 == 0 {
-			if !ok {
-				t.Fatal("key", i, "missing!")
-			}
-			if v.Size != i+5 {
-				t.Fatal("key", i, "size", v.Size)
-			}
-		} else if i%37 == 0 {
-			if ok && v.Size > 0 {
-				t.Fatal("key", i, "should have been deleted needle value", v)
-			}
-		} else if i%2 == 0 {
-			if v.Size != i {
-				t.Fatal("key", i, "size", v.Size)
-			}
-		}
-	}
-
-	for i := uint32(10 * batch); i < 100*batch; i++ {
-		v, ok := m.Get(Key(i))
-		if i%37 == 0 {
-			if ok && v.Size > 0 {
-				t.Fatal("key", i, "should have been deleted needle value", v)
-			}
-		} else if i%2 == 0 {
-			if v == nil {
-				t.Fatal("key", i, "missing")
-			}
-			if v.Size != i {
-				t.Fatal("key", i, "size", v.Size)
-			}
-		}
-	}
-
-}

go/storage/compress.go

@@ -1,59 +0,0 @@
-package storage
-
-import (
-	"bytes"
-	"compress/flate"
-	"compress/gzip"
-	"io/ioutil"
-	"strings"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-)
-
-/*
-* Default more not to gzip since gzip can be done on client side.
- */
-func IsGzippable(ext, mtype string) bool {
-	if strings.HasPrefix(mtype, "text/") {
-		return true
-	}
-	switch ext {
-	case ".zip", ".rar", ".gz", ".bz2", ".xz":
-		return false
-	case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
-		return true
-	}
-	if strings.HasPrefix(mtype, "application/") {
-		if strings.HasSuffix(mtype, "xml") {
-			return true
-		}
-		if strings.HasSuffix(mtype, "script") {
-			return true
-		}
-	}
-	return false
-}
-
-func GzipData(input []byte) ([]byte, error) {
-	buf := new(bytes.Buffer)
-	w, _ := gzip.NewWriterLevel(buf, flate.BestCompression)
-	if _, err := w.Write(input); err != nil {
-		glog.V(2).Infoln("error compressing data:", err)
-		return nil, err
-	}
-	if err := w.Close(); err != nil {
-		glog.V(2).Infoln("error closing compressed data:", err)
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-func UnGzipData(input []byte) ([]byte, error) {
-	buf := bytes.NewBuffer(input)
-	r, _ := gzip.NewReader(buf)
-	defer r.Close()
-	output, err := ioutil.ReadAll(r)
-	if err != nil {
-		glog.V(2).Infoln("error uncompressing data:", err)
-	}
-	return output, err
-}

go/storage/file_id.go

@@ -1,43 +0,0 @@
-package storage
-
-import (
-	"encoding/hex"
-	"errors"
-	"strings"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-type FileId struct {
-	VolumeId VolumeId
-	Key      uint64
-	Hashcode uint32
-}
-
-func NewFileIdFromNeedle(VolumeId VolumeId, n *Needle) *FileId {
-	return &FileId{VolumeId: VolumeId, Key: n.Id, Hashcode: n.Cookie}
-}
-func NewFileId(VolumeId VolumeId, Key uint64, Hashcode uint32) *FileId {
-	return &FileId{VolumeId: VolumeId, Key: Key, Hashcode: Hashcode}
-}
-func ParseFileId(fid string) (*FileId, error) {
-	a := strings.Split(fid, ",")
-	if len(a) != 2 {
-		glog.V(1).Infoln("Invalid fid ", fid, ", split length ", len(a))
-		return nil, errors.New("Invalid fid " + fid)
-	}
-	vid_string, key_hash_string := a[0], a[1]
-	volumeId, _ := NewVolumeId(vid_string)
-	key, hash, e := ParseKeyHash(key_hash_string)
-	return &FileId{VolumeId: volumeId, Key: key, Hashcode: hash}, e
-}
-func (n *FileId) String() string {
-	bytes := make([]byte, 12)
-	util.Uint64toBytes(bytes[0:8], n.Key)
-	util.Uint32toBytes(bytes[8:12], n.Hashcode)
-	nonzero_index := 0
-	for ; bytes[nonzero_index] == 0; nonzero_index++ {
-	}
-	return n.VolumeId.String() + "," + hex.EncodeToString(bytes[nonzero_index:])
-}

go/storage/needle.go

@@ -1,217 +0,0 @@
-package storage
-
-import (
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"mime"
-	"net/http"
-	"path"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/images"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-const (
-	NeedleHeaderSize      = 16 //should never change this
-	NeedlePaddingSize     = 8
-	NeedleChecksumSize    = 4
-	MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8
-)
-
-/*
-* A Needle means a uploaded and stored file.
-* Needle file size is limited to 4GB for now.
- */
-type Needle struct {
-	Cookie uint32 `comment:"random number to mitigate brute force lookups"`
-	Id     uint64 `comment:"needle id"`
-	Size   uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
-
-	DataSize     uint32 `comment:"Data size"` //version2
-	Data         []byte `comment:"The actual file data"`
-	Flags        byte   `comment:"boolean flags"` //version2
-	NameSize     uint8  //version2
-	Name         []byte `comment:"maximum 256 characters"` //version2
-	MimeSize     uint8  //version2
-	Mime         []byte `comment:"maximum 256 characters"` //version2
-	LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
-	Ttl          *TTL
-
-	Checksum CRC    `comment:"CRC32 to check integrity"`
-	Padding  []byte `comment:"Aligned to 8 bytes"`
-}
-
-func (n *Needle) String() (str string) {
-	str = fmt.Sprintf("Cookie:%d, Id:%d, Size:%d, DataSize:%d, Name: %s, Mime: %s", n.Cookie, n.Id, n.Size, n.DataSize, n.Name, n.Mime)
-	return
-}
-
-func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string, isGzipped bool, modifiedTime uint64, ttl *TTL, e error) {
-	form, fe := r.MultipartReader()
-	if fe != nil {
-		glog.V(0).Infoln("MultipartReader [ERROR]", fe)
-		e = fe
-		return
-	}
-
-	//first multi-part item
-	part, fe := form.NextPart()
-	if fe != nil {
-		glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
-		e = fe
-		return
-	}
-
-	fileName = part.FileName()
-	if fileName != "" {
-		fileName = path.Base(fileName)
-	}
-
-	data, e = ioutil.ReadAll(part)
-	if e != nil {
-		glog.V(0).Infoln("Reading Content [ERROR]", e)
-		return
-	}
-
-	//if the filename is empty string, do a search on the other multi-part items
-	for fileName == "" {
-		part2, fe := form.NextPart()
-		if fe != nil {
-			break // no more or on error, just safely break
-		}
-
-		fName := part2.FileName()
-
-		//found the first <file type> multi-part has filename
-		if fName != "" {
-			data2, fe2 := ioutil.ReadAll(part2)
-			if fe2 != nil {
-				glog.V(0).Infoln("Reading Content [ERROR]", fe2)
-				e = fe2
-				return
-			}
-
-			//update
-			data = data2
-			fileName = path.Base(fName)
-			break
-		}
-	}
-
-	dotIndex := strings.LastIndex(fileName, ".")
-	ext, mtype := "", ""
-	if dotIndex > 0 {
-		ext = strings.ToLower(fileName[dotIndex:])
-		mtype = mime.TypeByExtension(ext)
-	}
-	contentType := part.Header.Get("Content-Type")
-	if contentType != "" && mtype != contentType {
-		mimeType = contentType //only return mime type if not deductable
-		mtype = contentType
-	}
-	if part.Header.Get("Content-Encoding") == "gzip" {
-		isGzipped = true
-	} else if IsGzippable(ext, mtype) {
-		if data, e = GzipData(data); e != nil {
-			return
-		}
-		isGzipped = true
-	}
-	if ext == ".gz" {
-		isGzipped = true
-	}
-	if strings.HasSuffix(fileName, ".gz") &&
-		!strings.HasSuffix(fileName, ".tar.gz") {
-		fileName = fileName[:len(fileName)-3]
-	}
-	modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
-	ttl, _ = ReadTTL(r.FormValue("ttl"))
-	return
-}
-func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {
-	fname, mimeType, isGzipped := "", "", false
-	n = new(Needle)
-	fname, n.Data, mimeType, isGzipped, n.LastModified, n.Ttl, e = ParseUpload(r)
-	if e != nil {
-		return
-	}
-	if len(fname) < 256 {
-		n.Name = []byte(fname)
-		n.SetHasName()
-	}
-	if len(mimeType) < 256 {
-		n.Mime = []byte(mimeType)
-		n.SetHasMime()
-	}
-	if isGzipped {
-		n.SetGzipped()
-	}
-	if n.LastModified == 0 {
-		n.LastModified = uint64(time.Now().Unix())
-	}
-	n.SetHasLastModifiedDate()
-	if n.Ttl != EMPTY_TTL {
-		n.SetHasTtl()
-	}
-
-	if fixJpgOrientation {
-		loweredName := strings.ToLower(fname)
-		if mimeType == "image/jpeg" || strings.HasSuffix(loweredName, ".jpg") || strings.HasSuffix(loweredName, ".jpeg") {
-			n.Data = images.FixJpgOrientation(n.Data)
-		}
-	}
-
-	n.Checksum = NewCRC(n.Data)
-
-	commaSep := strings.LastIndex(r.URL.Path, ",")
-	dotSep := strings.LastIndex(r.URL.Path, ".")
-	fid := r.URL.Path[commaSep+1:]
-	if dotSep > 0 {
-		fid = r.URL.Path[commaSep+1 : dotSep]
-	}
-
-	e = n.ParsePath(fid)
-
-	return
-}
-func (n *Needle) ParsePath(fid string) (err error) {
-	length := len(fid)
-	if length <= 8 {
-		return errors.New("Invalid fid:" + fid)
-	}
-	delta := ""
-	deltaIndex := strings.LastIndex(fid, "_")
-	if deltaIndex > 0 {
-		fid, delta = fid[0:deltaIndex], fid[deltaIndex+1:]
-	}
-	n.Id, n.Cookie, err = ParseKeyHash(fid)
-	if err != nil {
-		return err
-	}
-	if delta != "" {
-		if d, e := strconv.ParseUint(delta, 10, 64); e == nil {
-			n.Id += d
-		} else {
-			return e
-		}
-	}
-	return err
-}
-
-func ParseKeyHash(key_hash_string string) (uint64, uint32, error) {
-	key_hash_bytes, khe := hex.DecodeString(key_hash_string)
-	key_hash_len := len(key_hash_bytes)
-	if khe != nil || key_hash_len <= 4 {
-		glog.V(0).Infoln("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe)
-		return 0, 0, errors.New("Invalid key and hash:" + key_hash_string)
-	}
-	key := util.BytesToUint64(key_hash_bytes[0 : key_hash_len-4])
-	hash := util.BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len])
-	return key, hash, nil
-}

go/storage/needle_map.go

@@ -1,68 +0,0 @@
-package storage
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-type NeedleMapType int
-
-const (
-	NeedleMapInMemory NeedleMapType = iota
-	NeedleMapLevelDb
-	NeedleMapBoltDb
-)
-
-type NeedleMapper interface {
-	Put(key uint64, offset uint32, size uint32) error
-	Get(key uint64) (element *NeedleValue, ok bool)
-	Delete(key uint64) error
-	Close()
-	Destroy() error
-	ContentSize() uint64
-	DeletedSize() uint64
-	FileCount() int
-	DeletedCount() int
-	MaxFileKey() uint64
-}
-
-type mapMetric struct {
-	DeletionCounter     int    `json:"DeletionCounter"`
-	FileCounter         int    `json:"FileCounter"`
-	DeletionByteCounter uint64 `json:"DeletionByteCounter"`
-	FileByteCounter     uint64 `json:"FileByteCounter"`
-	MaximumFileKey      uint64 `json:"MaxFileKey"`
-}
-
-func appendToIndexFile(indexFile *os.File,
-	key uint64, offset uint32, size uint32) error {
-	bytes := make([]byte, 16)
-	util.Uint64toBytes(bytes[0:8], key)
-	util.Uint32toBytes(bytes[8:12], offset)
-	util.Uint32toBytes(bytes[12:16], size)
-	if _, err := indexFile.Seek(0, 2); err != nil {
-		return fmt.Errorf("cannot seek end of indexfile %s: %v",
-			indexFile.Name(), err)
-	}
-	_, err := indexFile.Write(bytes)
-	return err
-}
-
-func (mm *mapMetric) logDelete(deletedByteCount uint32) {
-	mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(deletedByteCount)
-	mm.DeletionCounter++
-}
-
-func (mm *mapMetric) logPut(key uint64, oldSize uint32, newSize uint32) {
-	if key > mm.MaximumFileKey {
-		mm.MaximumFileKey = key
-	}
-	mm.FileCounter++
-	mm.FileByteCounter = mm.FileByteCounter + uint64(newSize)
-	if oldSize > 0 {
-		mm.DeletionCounter++
-		mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(oldSize)
-	}
-}

go/storage/needle_map_boltdb.go

@@ -1,181 +0,0 @@
-package storage
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/boltdb/bolt"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-type BoltDbNeedleMap struct {
-	dbFileName string
-	indexFile  *os.File
-	db         *bolt.DB
-	mapMetric
-}
-
-var boltdbBucket = []byte("weed")
-
-func NewBoltDbNeedleMap(dbFileName string, indexFile *os.File) (m *BoltDbNeedleMap, err error) {
-	m = &BoltDbNeedleMap{indexFile: indexFile, dbFileName: dbFileName}
-	if !isBoltDbFresh(dbFileName, indexFile) {
-		glog.V(1).Infof("Start to Generate %s from %s", dbFileName, indexFile.Name())
-		generateBoltDbFile(dbFileName, indexFile)
-		glog.V(1).Infof("Finished Generating %s from %s", dbFileName, indexFile.Name())
-	}
-	glog.V(1).Infof("Opening %s...", dbFileName)
-	if m.db, err = bolt.Open(dbFileName, 0644, nil); err != nil {
-		return
-	}
-	glog.V(1).Infof("Loading %s...", indexFile.Name())
-	nm, indexLoadError := LoadNeedleMap(indexFile)
-	if indexLoadError != nil {
-		return nil, indexLoadError
-	}
-	m.mapMetric = nm.mapMetric
-	return
-}
-
-func isBoltDbFresh(dbFileName string, indexFile *os.File) bool {
-	// normally we always write to index file first
-	dbLogFile, err := os.Open(dbFileName)
-	if err != nil {
-		return false
-	}
-	defer dbLogFile.Close()
-	dbStat, dbStatErr := dbLogFile.Stat()
-	indexStat, indexStatErr := indexFile.Stat()
-	if dbStatErr != nil || indexStatErr != nil {
-		glog.V(0).Infof("Can not stat file: %v and %v", dbStatErr, indexStatErr)
-		return false
-	}
-
-	return dbStat.ModTime().After(indexStat.ModTime())
-}
-
-func generateBoltDbFile(dbFileName string, indexFile *os.File) error {
-	db, err := bolt.Open(dbFileName, 0644, nil)
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-	return WalkIndexFile(indexFile, func(key uint64, offset, size uint32) error {
-		if offset > 0 {
-			boltDbWrite(db, key, offset, size)
-		} else {
-			boltDbDelete(db, key)
-		}
-		return nil
-	})
-}
-
-func (m *BoltDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
-	bytes := make([]byte, 8)
-	var data []byte
-	util.Uint64toBytes(bytes, key)
-	err := m.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket(boltdbBucket)
-		if bucket == nil {
-			return fmt.Errorf("Bucket %q not found!", boltdbBucket)
-		}
-
-		data = bucket.Get(bytes)
-		return nil
-	})
-
-	if err != nil || len(data) != 8 {
-		return nil, false
-	}
-	offset := util.BytesToUint32(data[0:4])
-	size := util.BytesToUint32(data[4:8])
-	return &NeedleValue{Key: Key(key), Offset: offset, Size: size}, true
-}
-
-func (m *BoltDbNeedleMap) Put(key uint64, offset uint32, size uint32) error {
-	var oldSize uint32
-	if oldNeedle, ok := m.Get(key); ok {
-		oldSize = oldNeedle.Size
-	}
-	m.logPut(key, oldSize, size)
-	// write to index file first
-	if err := appendToIndexFile(m.indexFile, key, offset, size); err != nil {
-		return fmt.Errorf("cannot write to indexfile %s: %v", m.indexFile.Name(), err)
-	}
-	return boltDbWrite(m.db, key, offset, size)
-}
-
-func boltDbWrite(db *bolt.DB,
-	key uint64, offset uint32, size uint32) error {
-	bytes := make([]byte, 16)
-	util.Uint64toBytes(bytes[0:8], key)
-	util.Uint32toBytes(bytes[8:12], offset)
-	util.Uint32toBytes(bytes[12:16], size)
-	return db.Update(func(tx *bolt.Tx) error {
-		bucket, err := tx.CreateBucketIfNotExists(boltdbBucket)
-		if err != nil {
-			return err
-		}
-
-		err = bucket.Put(bytes[0:8], bytes[8:16])
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-func boltDbDelete(db *bolt.DB, key uint64) error {
-	bytes := make([]byte, 8)
-	util.Uint64toBytes(bytes, key)
-	return db.Update(func(tx *bolt.Tx) error {
-		bucket, err := tx.CreateBucketIfNotExists(boltdbBucket)
-		if err != nil {
-			return err
-		}
-
-		err = bucket.Delete(bytes)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-func (m *BoltDbNeedleMap) Delete(key uint64) error {
-	if oldNeedle, ok := m.Get(key); ok {
-		m.logDelete(oldNeedle.Size)
-	}
-	// write to index file first
-	if err := appendToIndexFile(m.indexFile, key, 0, 0); err != nil {
-		return err
-	}
-	return boltDbDelete(m.db, key)
-}
-
-func (m *BoltDbNeedleMap) Close() {
-	m.db.Close()
-}
-
-func (m *BoltDbNeedleMap) Destroy() error {
-	m.Close()
-	os.Remove(m.indexFile.Name())
-	return os.Remove(m.dbFileName)
-}
-
-func (m *BoltDbNeedleMap) ContentSize() uint64 {
-	return m.FileByteCounter
-}
-func (m *BoltDbNeedleMap) DeletedSize() uint64 {
-	return m.DeletionByteCounter
-}
-func (m *BoltDbNeedleMap) FileCount() int {
-	return m.FileCounter
-}
-func (m *BoltDbNeedleMap) DeletedCount() int {
-	return m.DeletionCounter
-}
-func (m *BoltDbNeedleMap) MaxFileKey() uint64 {
-	return m.MaximumFileKey
-}

go/storage/needle_map_leveldb.go

@@ -1,150 +0,0 @@
-package storage
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
-	"github.com/syndtr/goleveldb/leveldb"
-)
-
-type LevelDbNeedleMap struct {
-	dbFileName string
-	indexFile  *os.File
-	db         *leveldb.DB
-	mapMetric
-}
-
-func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File) (m *LevelDbNeedleMap, err error) {
-	m = &LevelDbNeedleMap{indexFile: indexFile, dbFileName: dbFileName}
-	if !isLevelDbFresh(dbFileName, indexFile) {
-		glog.V(1).Infof("Start to Generate %s from %s", dbFileName, indexFile.Name())
-		generateLevelDbFile(dbFileName, indexFile)
-		glog.V(1).Infof("Finished Generating %s from %s", dbFileName, indexFile.Name())
-	}
-	glog.V(1).Infof("Opening %s...", dbFileName)
-	if m.db, err = leveldb.OpenFile(dbFileName, nil); err != nil {
-		return
-	}
-	glog.V(1).Infof("Loading %s...", indexFile.Name())
-	nm, indexLoadError := LoadNeedleMap(indexFile)
-	if indexLoadError != nil {
-		return nil, indexLoadError
-	}
-	m.mapMetric = nm.mapMetric
-	return
-}
-
-func isLevelDbFresh(dbFileName string, indexFile *os.File) bool {
-	// normally we always write to index file first
-	dbLogFile, err := os.Open(filepath.Join(dbFileName, "LOG"))
-	if err != nil {
-		return false
-	}
-	defer dbLogFile.Close()
-	dbStat, dbStatErr := dbLogFile.Stat()
-	indexStat, indexStatErr := indexFile.Stat()
-	if dbStatErr != nil || indexStatErr != nil {
-		glog.V(0).Infof("Can not stat file: %v and %v", dbStatErr, indexStatErr)
-		return false
-	}
-
-	return dbStat.ModTime().After(indexStat.ModTime())
-}
-
-func generateLevelDbFile(dbFileName string, indexFile *os.File) error {
-	db, err := leveldb.OpenFile(dbFileName, nil)
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-	return WalkIndexFile(indexFile, func(key uint64, offset, size uint32) error {
-		if offset > 0 {
-			levelDbWrite(db, key, offset, size)
-		} else {
-			levelDbDelete(db, key)
-		}
-		return nil
-	})
-}
-
-func (m *LevelDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
-	bytes := make([]byte, 8)
-	util.Uint64toBytes(bytes, key)
-	data, err := m.db.Get(bytes, nil)
-	if err != nil || len(data) != 8 {
-		return nil, false
-	}
-	offset := util.BytesToUint32(data[0:4])
-	size := util.BytesToUint32(data[4:8])
-	return &NeedleValue{Key: Key(key), Offset: offset, Size: size}, true
-}
-
-func (m *LevelDbNeedleMap) Put(key uint64, offset uint32, size uint32) error {
-	var oldSize uint32
-	if oldNeedle, ok := m.Get(key); ok {
-		oldSize = oldNeedle.Size
-	}
-	m.logPut(key, oldSize, size)
-	// write to index file first
-	if err := appendToIndexFile(m.indexFile, key, offset, size); err != nil {
-		return fmt.Errorf("cannot write to indexfile %s: %v", m.indexFile.Name(), err)
-	}
-	return levelDbWrite(m.db, key, offset, size)
-}
-
-func levelDbWrite(db *leveldb.DB,
-	key uint64, offset uint32, size uint32) error {
-	bytes := make([]byte, 16)
-	util.Uint64toBytes(bytes[0:8], key)
-	util.Uint32toBytes(bytes[8:12], offset)
-	util.Uint32toBytes(bytes[12:16], size)
-	if err := db.Put(bytes[0:8], bytes[8:16], nil); err != nil {
-		return fmt.Errorf("failed to write leveldb: %v", err)
-	}
-	return nil
-}
-func levelDbDelete(db *leveldb.DB, key uint64) error {
-	bytes := make([]byte, 8)
-	util.Uint64toBytes(bytes, key)
-	return db.Delete(bytes, nil)
-}
-
-func (m *LevelDbNeedleMap) Delete(key uint64) error {
-	if oldNeedle, ok := m.Get(key); ok {
-		m.logDelete(oldNeedle.Size)
-	}
-	// write to index file first
-	if err := appendToIndexFile(m.indexFile, key, 0, 0); err != nil {
-		return err
-	}
-	return levelDbDelete(m.db, key)
-}
-
-func (m *LevelDbNeedleMap) Close() {
-	m.db.Close()
-}
-
-func (m *LevelDbNeedleMap) Destroy() error {
-	m.Close()
-	os.Remove(m.indexFile.Name())
-	return os.Remove(m.dbFileName)
-}
-
-func (m *LevelDbNeedleMap) ContentSize() uint64 {
-	return m.FileByteCounter
-}
-func (m *LevelDbNeedleMap) DeletedSize() uint64 {
-	return m.DeletionByteCounter
-}
-func (m *LevelDbNeedleMap) FileCount() int {
-	return m.FileCounter
-}
-func (m *LevelDbNeedleMap) DeletedCount() int {
-	return m.DeletionCounter
-}
-func (m *LevelDbNeedleMap) MaxFileKey() uint64 {
-	return m.MaximumFileKey
-}

go/storage/needle_map_memory.go

@@ -1,126 +0,0 @@
-package storage
-
-import (
-	"io"
-	"os"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-type NeedleMap struct {
-	indexFile *os.File
-	m         CompactMap
-
-	mapMetric
-}
-
-func NewNeedleMap(file *os.File) *NeedleMap {
-	nm := &NeedleMap{
-		m:         NewCompactMap(),
-		indexFile: file,
-	}
-	return nm
-}
-
-const (
-	RowsToRead = 1024
-)
-
-func LoadNeedleMap(file *os.File) (*NeedleMap, error) {
-	nm := NewNeedleMap(file)
-	e := WalkIndexFile(file, func(key uint64, offset, size uint32) error {
-		if key > nm.MaximumFileKey {
-			nm.MaximumFileKey = key
-		}
-		nm.FileCounter++
-		nm.FileByteCounter = nm.FileByteCounter + uint64(size)
-		if offset > 0 {
-			oldSize := nm.m.Set(Key(key), offset, size)
-			glog.V(3).Infoln("reading key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize)
-			if oldSize > 0 {
-				nm.DeletionCounter++
-				nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
-			}
-		} else {
-			oldSize := nm.m.Delete(Key(key))
-			glog.V(3).Infoln("removing key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize)
-			nm.DeletionCounter++
-			nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
-		}
-		return nil
-	})
-	glog.V(1).Infoln("max file key:", nm.MaximumFileKey)
-	return nm, e
-}
-
-// walks through the index file, calls fn function with each key, offset, size
-// stops with the error returned by the fn function
-func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) error {
-	var readerOffset int64
-	bytes := make([]byte, 16*RowsToRead)
-	count, e := r.ReadAt(bytes, readerOffset)
-	glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
-	readerOffset += int64(count)
-	var (
-		key          uint64
-		offset, size uint32
-		i            int
-	)
-
-	for count > 0 && e == nil || e == io.EOF {
-		for i = 0; i+16 <= count; i += 16 {
-			key = util.BytesToUint64(bytes[i : i+8])
-			offset = util.BytesToUint32(bytes[i+8 : i+12])
-			size = util.BytesToUint32(bytes[i+12 : i+16])
-			if e = fn(key, offset, size); e != nil {
-				return e
-			}
-		}
-		if e == io.EOF {
-			return nil
-		}
-		count, e = r.ReadAt(bytes, readerOffset)
-		glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
-		readerOffset += int64(count)
-	}
-	return e
-}
-
-func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) error {
-	oldSize := nm.m.Set(Key(key), offset, size)
-	nm.logPut(key, oldSize, size)
-	return appendToIndexFile(nm.indexFile, key, offset, size)
-}
-func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
-	element, ok = nm.m.Get(Key(key))
-	return
-}
-func (nm *NeedleMap) Delete(key uint64) error {
-	deletedBytes := nm.m.Delete(Key(key))
-	nm.logDelete(deletedBytes)
-	return appendToIndexFile(nm.indexFile, key, 0, 0)
-}
-func (nm *NeedleMap) Close() {
-	_ = nm.indexFile.Close()
-}
-func (nm *NeedleMap) Destroy() error {
-	nm.Close()
-	return os.Remove(nm.indexFile.Name())
-}
-func (nm NeedleMap) ContentSize() uint64 {
-	return nm.FileByteCounter
-}
-func (nm NeedleMap) DeletedSize() uint64 {
-	return nm.DeletionByteCounter
-}
-func (nm NeedleMap) FileCount() int {
-	return nm.FileCounter
-}
-func (nm NeedleMap) DeletedCount() int {
-	return nm.DeletionCounter
-}
-
-func (nm NeedleMap) MaxFileKey() uint64 {
-	return nm.MaximumFileKey
-}

go/storage/needle_read_write.go

@@ -1,294 +0,0 @@
-package storage
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-const (
-	FlagGzip                = 0x01
-	FlagHasName             = 0x02
-	FlagHasMime             = 0x04
-	FlagHasLastModifiedDate = 0x08
-	FlagHasTtl              = 0x10
-	LastModifiedBytesLength = 5
-	TtlBytesLength          = 2
-)
-
-func (n *Needle) DiskSize() int64 {
-	padding := NeedlePaddingSize - ((NeedleHeaderSize + int64(n.Size) + NeedleChecksumSize) % NeedlePaddingSize)
-	return NeedleHeaderSize + int64(n.Size) + padding + NeedleChecksumSize
-}
-func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) {
-	if s, ok := w.(io.Seeker); ok {
-		if end, e := s.Seek(0, 1); e == nil {
-			defer func(s io.Seeker, off int64) {
-				if err != nil {
-					if _, e = s.Seek(off, 0); e != nil {
-						glog.V(0).Infof("Failed to seek %s back to %d with error: %v", w, off, e)
-					}
-				}
-			}(s, end)
-		} else {
-			err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
-			return
-		}
-	}
-	switch version {
-	case Version1:
-		header := make([]byte, NeedleHeaderSize)
-		util.Uint32toBytes(header[0:4], n.Cookie)
-		util.Uint64toBytes(header[4:12], n.Id)
-		n.Size = uint32(len(n.Data))
-		size = n.Size
-		util.Uint32toBytes(header[12:16], n.Size)
-		if _, err = w.Write(header); err != nil {
-			return
-		}
-		if _, err = w.Write(n.Data); err != nil {
-			return
-		}
-		padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
-		util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
-		_, err = w.Write(header[0 : NeedleChecksumSize+padding])
-		return
-	case Version2:
-		header := make([]byte, NeedleHeaderSize)
-		util.Uint32toBytes(header[0:4], n.Cookie)
-		util.Uint64toBytes(header[4:12], n.Id)
-		n.DataSize, n.NameSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Name)), uint8(len(n.Mime))
-		if n.DataSize > 0 {
-			n.Size = 4 + n.DataSize + 1
-			if n.HasName() {
-				n.Size = n.Size + 1 + uint32(n.NameSize)
-			}
-			if n.HasMime() {
-				n.Size = n.Size + 1 + uint32(n.MimeSize)
-			}
-			if n.HasLastModifiedDate() {
-				n.Size = n.Size + LastModifiedBytesLength
-			}
-			if n.HasTtl() {
-				n.Size = n.Size + TtlBytesLength
-			}
-		} else {
-			n.Size = 0
-		}
-		size = n.DataSize
-		util.Uint32toBytes(header[12:16], n.Size)
-		if _, err = w.Write(header); err != nil {
-			return
-		}
-		if n.DataSize > 0 {
-			util.Uint32toBytes(header[0:4], n.DataSize)
-			if _, err = w.Write(header[0:4]); err != nil {
-				return
-			}
-			if _, err = w.Write(n.Data); err != nil {
-				return
-			}
-			util.Uint8toBytes(header[0:1], n.Flags)
-			if _, err = w.Write(header[0:1]); err != nil {
-				return
-			}
-			if n.HasName() {
-				util.Uint8toBytes(header[0:1], n.NameSize)
-				if _, err = w.Write(header[0:1]); err != nil {
-					return
-				}
-				if _, err = w.Write(n.Name); err != nil {
-					return
-				}
-			}
-			if n.HasMime() {
-				util.Uint8toBytes(header[0:1], n.MimeSize)
-				if _, err = w.Write(header[0:1]); err != nil {
-					return
-				}
-				if _, err = w.Write(n.Mime); err != nil {
-					return
-				}
-			}
-			if n.HasLastModifiedDate() {
-				util.Uint64toBytes(header[0:8], n.LastModified)
-				if _, err = w.Write(header[8-LastModifiedBytesLength : 8]); err != nil {
-					return
-				}
-			}
-			if n.HasTtl() {
-				n.Ttl.ToBytes(header[0:TtlBytesLength])
-				if _, err = w.Write(header[0:TtlBytesLength]); err != nil {
-					return
-				}
-			}
-		}
-		padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
-		util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
-		_, err = w.Write(header[0 : NeedleChecksumSize+padding])
-		return n.DataSize, err
-	}
-	return 0, fmt.Errorf("Unsupported Version! (%d)", version)
-}
-
-func (n *Needle) Read(r *os.File, offset int64, size uint32, version Version) (ret int, err error) {
-	switch version {
-	case Version1:
-		bytes := make([]byte, NeedleHeaderSize+size+NeedleChecksumSize)
-		if ret, err = r.ReadAt(bytes, offset); err != nil {
-			return
-		}
-		n.readNeedleHeader(bytes)
-		n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size]
-		checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize])
-		newChecksum := NewCRC(n.Data)
-		if checksum != newChecksum.Value() {
-			return 0, errors.New("CRC error! Data On Disk Corrupted")
-		}
-		n.Checksum = newChecksum
-		return
-	case Version2:
-		if size == 0 {
-			return 0, nil
-		}
-		bytes := make([]byte, NeedleHeaderSize+size+NeedleChecksumSize)
-		if ret, err = r.ReadAt(bytes, offset); err != nil {
-			return
-		}
-		if ret != int(NeedleHeaderSize+size+NeedleChecksumSize) {
-			return 0, errors.New("File Entry Not Found")
-		}
-		n.readNeedleHeader(bytes)
-		if n.Size != size {
-			return 0, fmt.Errorf("File Entry Not Found. Needle %d Memory %d", n.Size, size)
-		}
-		n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)])
-		checksum := util.BytesToUint32(bytes[NeedleHeaderSize+n.Size : NeedleHeaderSize+n.Size+NeedleChecksumSize])
-		newChecksum := NewCRC(n.Data)
-		if checksum != newChecksum.Value() {
-			return 0, errors.New("CRC Found Data On Disk Corrupted")
-		}
-		n.Checksum = newChecksum
-		return
-	}
-	return 0, fmt.Errorf("Unsupported Version! (%d)", version)
-}
-func (n *Needle) readNeedleHeader(bytes []byte) {
-	n.Cookie = util.BytesToUint32(bytes[0:4])
-	n.Id = util.BytesToUint64(bytes[4:12])
-	n.Size = util.BytesToUint32(bytes[12:NeedleHeaderSize])
-}
-func (n *Needle) readNeedleDataVersion2(bytes []byte) {
-	index, lenBytes := 0, len(bytes)
-	if index < lenBytes {
-		n.DataSize = util.BytesToUint32(bytes[index : index+4])
-		index = index + 4
-		if int(n.DataSize)+index > lenBytes {
-			// this if clause is due to bug #87 and #93, fixed in v0.69
-			// remove this clause later
-			return
-		}
-		n.Data = bytes[index : index+int(n.DataSize)]
-		index = index + int(n.DataSize)
-		n.Flags = bytes[index]
-		index = index + 1
-	}
-	if index < lenBytes && n.HasName() {
-		n.NameSize = uint8(bytes[index])
-		index = index + 1
-		n.Name = bytes[index : index+int(n.NameSize)]
-		index = index + int(n.NameSize)
-	}
-	if index < lenBytes && n.HasMime() {
-		n.MimeSize = uint8(bytes[index])
-		index = index + 1
-		n.Mime = bytes[index : index+int(n.MimeSize)]
-		index = index + int(n.MimeSize)
-	}
-	if index < lenBytes && n.HasLastModifiedDate() {
-		n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
-		index = index + LastModifiedBytesLength
-	}
-	if index < lenBytes && n.HasTtl() {
-		n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
-		index = index + TtlBytesLength
-	}
-}
-
-func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, bodyLength uint32, err error) {
-	n = new(Needle)
-	if version == Version1 || version == Version2 {
-		bytes := make([]byte, NeedleHeaderSize)
-		var count int
-		count, err = r.ReadAt(bytes, offset)
-		if count <= 0 || err != nil {
-			return nil, 0, err
-		}
-		n.readNeedleHeader(bytes)
-		padding := NeedlePaddingSize - ((n.Size + NeedleHeaderSize + NeedleChecksumSize) % NeedlePaddingSize)
-		bodyLength = n.Size + NeedleChecksumSize + padding
-	}
-	return
-}
-
-//n should be a needle already read the header
-//the input stream will read until next file entry
-func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyLength uint32) (err error) {
-	if bodyLength <= 0 {
-		return nil
-	}
-	switch version {
-	case Version1:
-		bytes := make([]byte, bodyLength)
-		if _, err = r.ReadAt(bytes, offset); err != nil {
-			return
-		}
-		n.Data = bytes[:n.Size]
-		n.Checksum = NewCRC(n.Data)
-	case Version2:
-		bytes := make([]byte, bodyLength)
-		if _, err = r.ReadAt(bytes, offset); err != nil {
-			return
-		}
-		n.readNeedleDataVersion2(bytes[0:n.Size])
-		n.Checksum = NewCRC(n.Data)
-	default:
-		err = fmt.Errorf("Unsupported Version! (%d)", version)
-	}
-	return
-}
-
-func (n *Needle) IsGzipped() bool {
-	return n.Flags&FlagGzip > 0
-}
-func (n *Needle) SetGzipped() {
-	n.Flags = n.Flags | FlagGzip
-}
-func (n *Needle) HasName() bool {
-	return n.Flags&FlagHasName > 0
-}
-func (n *Needle) SetHasName() {
-	n.Flags = n.Flags | FlagHasName
-}
-func (n *Needle) HasMime() bool {
-	return n.Flags&FlagHasMime > 0
-}
-func (n *Needle) SetHasMime() {
-	n.Flags = n.Flags | FlagHasMime
-}
-func (n *Needle) HasLastModifiedDate() bool {
-	return n.Flags&FlagHasLastModifiedDate > 0
-}
-func (n *Needle) SetHasLastModifiedDate() {
-	n.Flags = n.Flags | FlagHasLastModifiedDate
-}
-func (n *Needle) HasTtl() bool {
-	return n.Flags&FlagHasTtl > 0
-}
-func (n *Needle) SetHasTtl() {
-	n.Flags = n.Flags | FlagHasTtl
-}

go/storage/replica_placement_test.go

@@ -1,14 +0,0 @@
-package storage
-
-import (
-	"testing"
-)
-
-func TestReplicaPlacemnetSerialDeserial(t *testing.T) {
-	rp, _ := NewReplicaPlacementFromString("001")
-	new_rp, _ := NewReplicaPlacementFromByte(rp.Byte())
-	if rp.String() != new_rp.String() {
-		println("expected:", rp.String(), "actual:", new_rp.String())
-		t.Fail()
-	}
-}

go/storage/store.go

@@ -1,376 +0,0 @@
-package storage
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"math/rand"
-	"strconv"
-	"strings"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/operation"
-	"github.com/chrislusf/weed-fs/go/security"
-	"github.com/chrislusf/weed-fs/go/util"
-	"github.com/golang/protobuf/proto"
-)
-
-const (
-	MAX_TTL_VOLUME_REMOVAL_DELAY = 10 // 10 minutes
-)
-
-type DiskLocation struct {
-	Directory      string
-	MaxVolumeCount int
-	volumes        map[VolumeId]*Volume
-}
-
-func (mn *DiskLocation) reset() {
-}
-
-type MasterNodes struct {
-	nodes    []string
-	lastNode int
-}
-
-func (mn *MasterNodes) String() string {
-	return fmt.Sprintf("nodes:%v, lastNode:%d", mn.nodes, mn.lastNode)
-}
-
-func NewMasterNodes(bootstrapNode string) (mn *MasterNodes) {
-	mn = &MasterNodes{nodes: []string{bootstrapNode}, lastNode: -1}
-	return
-}
-func (mn *MasterNodes) reset() {
-	if len(mn.nodes) > 1 && mn.lastNode > 0 {
-		mn.lastNode = -mn.lastNode
-	}
-}
-func (mn *MasterNodes) findMaster() (string, error) {
-	if len(mn.nodes) == 0 {
-		return "", errors.New("No master node found!")
-	}
-	if mn.lastNode < 0 {
-		for _, m := range mn.nodes {
-			if masters, e := operation.ListMasters(m); e == nil {
-				if len(masters) == 0 {
-					continue
-				}
-				mn.nodes = masters
-				mn.lastNode = rand.Intn(len(mn.nodes))
-				glog.V(2).Info("current master node is :", mn.nodes[mn.lastNode])
-				break
-			}
-		}
-	}
-	if mn.lastNode < 0 {
-		return "", errors.New("No master node available!")
-	}
-	return mn.nodes[mn.lastNode], nil
-}
-
-/*
- * A VolumeServer contains one Store
- */
-type Store struct {
-	Ip              string
-	Port            int
-	PublicUrl       string
-	Locations       []*DiskLocation
-	dataCenter      string //optional informaton, overwriting master setting if exists
-	rack            string //optional information, overwriting master setting if exists
-	connected       bool
-	volumeSizeLimit uint64 //read from the master
-	masterNodes     *MasterNodes
-}
-
-func (s *Store) String() (str string) {
-	str = fmt.Sprintf("Ip:%s, Port:%d, PublicUrl:%s, dataCenter:%s, rack:%s, connected:%v, volumeSizeLimit:%d, masterNodes:%s", s.Ip, s.Port, s.PublicUrl, s.dataCenter, s.rack, s.connected, s.volumeSizeLimit, s.masterNodes)
-	return
-}
-
-func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, needleMapKind NeedleMapType) (s *Store) {
-	s = &Store{Port: port, Ip: ip, PublicUrl: publicUrl}
-	s.Locations = make([]*DiskLocation, 0)
-	for i := 0; i < len(dirnames); i++ {
-		location := &DiskLocation{Directory: dirnames[i], MaxVolumeCount: maxVolumeCounts[i]}
-		location.volumes = make(map[VolumeId]*Volume)
-		location.loadExistingVolumes(needleMapKind)
-		s.Locations = append(s.Locations, location)
-	}
-	return
-}
-func (s *Store) AddVolume(volumeListString string, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string) error {
-	rt, e := NewReplicaPlacementFromString(replicaPlacement)
-	if e != nil {
-		return e
-	}
-	ttl, e := ReadTTL(ttlString)
-	if e != nil {
-		return e
-	}
-	for _, range_string := range strings.Split(volumeListString, ",") {
-		if strings.Index(range_string, "-") < 0 {
-			id_string := range_string
-			id, err := NewVolumeId(id_string)
-			if err != nil {
-				return fmt.Errorf("Volume Id %s is not a valid unsigned integer!", id_string)
-			}
-			e = s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl)
-		} else {
-			pair := strings.Split(range_string, "-")
-			start, start_err := strconv.ParseUint(pair[0], 10, 64)
-			if start_err != nil {
-				return fmt.Errorf("Volume Start Id %s is not a valid unsigned integer!", pair[0])
-			}
-			end, end_err := strconv.ParseUint(pair[1], 10, 64)
-			if end_err != nil {
-				return fmt.Errorf("Volume End Id %s is not a valid unsigned integer!", pair[1])
-			}
-			for id := start; id <= end; id++ {
-				if err := s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl); err != nil {
-					e = err
-				}
-			}
-		}
-	}
-	return e
-}
-func (s *Store) DeleteCollection(collection string) (e error) {
-	for _, location := range s.Locations {
-		for k, v := range location.volumes {
-			if v.Collection == collection {
-				e = v.Destroy()
-				if e != nil {
-					return
-				}
-				delete(location.volumes, k)
-			}
-		}
-	}
-	return
-}
-func (s *Store) DeleteVolume(volumes map[VolumeId]*Volume, v *Volume) (e error) {
-	e = v.Destroy()
-	if e != nil {
-		return
-	}
-	delete(volumes, v.Id)
-	return
-}
-func (s *Store) findVolume(vid VolumeId) *Volume {
-	for _, location := range s.Locations {
-		if v, found := location.volumes[vid]; found {
-			return v
-		}
-	}
-	return nil
-}
-func (s *Store) findFreeLocation() (ret *DiskLocation) {
-	max := 0
-	for _, location := range s.Locations {
-		currentFreeCount := location.MaxVolumeCount - len(location.volumes)
-		if currentFreeCount > max {
-			max = currentFreeCount
-			ret = location
-		}
-	}
-	return ret
-}
-func (s *Store) addVolume(vid VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL) error {
-	if s.findVolume(vid) != nil {
-		return fmt.Errorf("Volume Id %d already exists!", vid)
-	}
-	if location := s.findFreeLocation(); location != nil {
-		glog.V(0).Infof("In dir %s adds volume:%v collection:%s replicaPlacement:%v ttl:%v",
-			location.Directory, vid, collection, replicaPlacement, ttl)
-		if volume, err := NewVolume(location.Directory, collection, vid, needleMapKind, replicaPlacement, ttl); err == nil {
-			location.volumes[vid] = volume
-			return nil
-		} else {
-			return err
-		}
-	}
-	return fmt.Errorf("No more free space left")
-}
-
-func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) {
-	if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
-		for _, dir := range dirs {
-			name := dir.Name()
-			if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
-				collection := ""
-				base := name[:len(name)-len(".dat")]
-				i := strings.Index(base, "_")
-				if i > 0 {
-					collection, base = base[0:i], base[i+1:]
-				}
-				if vid, err := NewVolumeId(base); err == nil {
-					if l.volumes[vid] == nil {
-						if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil); e == nil {
-							l.volumes[vid] = v
-							glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s", l.Directory+"/"+name, v.ReplicaPlacement, v.Version(), v.Size(), v.Ttl.String())
-						}
-					}
-				}
-			}
-		}
-	}
-	glog.V(0).Infoln("Store started on dir:", l.Directory, "with", len(l.volumes), "volumes", "max", l.MaxVolumeCount)
-}
-func (s *Store) Status() []*VolumeInfo {
-	var stats []*VolumeInfo
-	for _, location := range s.Locations {
-		for k, v := range location.volumes {
-			s := &VolumeInfo{Id: VolumeId(k), Size: v.ContentSize(),
-				Collection:       v.Collection,
-				ReplicaPlacement: v.ReplicaPlacement,
-				Version:          v.Version(),
-				FileCount:        v.nm.FileCount(),
-				DeleteCount:      v.nm.DeletedCount(),
-				DeletedByteCount: v.nm.DeletedSize(),
-				ReadOnly:         v.readOnly,
-				Ttl:              v.Ttl}
-			stats = append(stats, s)
-		}
-	}
-	return stats
-}
-
-func (s *Store) SetDataCenter(dataCenter string) {
-	s.dataCenter = dataCenter
-}
-func (s *Store) SetRack(rack string) {
-	s.rack = rack
-}
-
-func (s *Store) SetBootstrapMaster(bootstrapMaster string) {
-	s.masterNodes = NewMasterNodes(bootstrapMaster)
-}
-func (s *Store) SendHeartbeatToMaster() (masterNode string, secretKey security.Secret, e error) {
-	masterNode, e = s.masterNodes.findMaster()
-	if e != nil {
-		return
-	}
-	var volumeMessages []*operation.VolumeInformationMessage
-	maxVolumeCount := 0
-	var maxFileKey uint64
-	for _, location := range s.Locations {
-		maxVolumeCount = maxVolumeCount + location.MaxVolumeCount
-		for k, v := range location.volumes {
-			if maxFileKey < v.nm.MaxFileKey() {
-				maxFileKey = v.nm.MaxFileKey()
-			}
-			if !v.expired(s.volumeSizeLimit) {
-				volumeMessage := &operation.VolumeInformationMessage{
-					Id:               proto.Uint32(uint32(k)),
-					Size:             proto.Uint64(uint64(v.Size())),
-					Collection:       proto.String(v.Collection),
-					FileCount:        proto.Uint64(uint64(v.nm.FileCount())),
-					DeleteCount:      proto.Uint64(uint64(v.nm.DeletedCount())),
-					DeletedByteCount: proto.Uint64(v.nm.DeletedSize()),
-					ReadOnly:         proto.Bool(v.readOnly),
-					ReplicaPlacement: proto.Uint32(uint32(v.ReplicaPlacement.Byte())),
-					Version:          proto.Uint32(uint32(v.Version())),
-					Ttl:              proto.Uint32(v.Ttl.ToUint32()),
-				}
-				volumeMessages = append(volumeMessages, volumeMessage)
-			} else {
-				if v.exiredLongEnough(MAX_TTL_VOLUME_REMOVAL_DELAY) {
-					s.DeleteVolume(location.volumes, v)
-					glog.V(0).Infoln("volume", v.Id, "is deleted.")
-				} else {
-					glog.V(0).Infoln("volume", v.Id, "is expired.")
-				}
-			}
-		}
-	}
-
-	joinMessage := &operation.JoinMessage{
-		IsInit:         proto.Bool(!s.connected),
-		Ip:             proto.String(s.Ip),
-		Port:           proto.Uint32(uint32(s.Port)),
-		PublicUrl:      proto.String(s.PublicUrl),
-		MaxVolumeCount: proto.Uint32(uint32(maxVolumeCount)),
-		MaxFileKey:     proto.Uint64(maxFileKey),
-		DataCenter:     proto.String(s.dataCenter),
-		Rack:           proto.String(s.rack),
-		Volumes:        volumeMessages,
-	}
-
-	data, err := proto.Marshal(joinMessage)
-	if err != nil {
-		return "", "", err
-	}
-
-	joinUrl := "http://" + masterNode + "/dir/join"
-
-	jsonBlob, err := util.PostBytes(joinUrl, data)
-	if err != nil {
-		s.masterNodes.reset()
-		return "", "", err
-	}
-	var ret operation.JoinResult
-	if err := json.Unmarshal(jsonBlob, &ret); err != nil {
-		glog.V(0).Infof("Failed to join %s with response: %s", joinUrl, string(jsonBlob))
-		return masterNode, "", err
-	}
-	if ret.Error != "" {
-		return masterNode, "", errors.New(ret.Error)
-	}
-	s.volumeSizeLimit = ret.VolumeSizeLimit
-	secretKey = security.Secret(ret.SecretKey)
-	s.connected = true
-	return
-}
-func (s *Store) Close() {
-	for _, location := range s.Locations {
-		for _, v := range location.volumes {
-			v.Close()
-		}
-	}
-}
-func (s *Store) Write(i VolumeId, n *Needle) (size uint32, err error) {
-	if v := s.findVolume(i); v != nil {
-		if v.readOnly {
-			err = fmt.Errorf("Volume %d is read only", i)
-			return
-		}
-		if MaxPossibleVolumeSize >= v.ContentSize()+uint64(size) {
-			size, err = v.write(n)
-		} else {
-			err = fmt.Errorf("Volume Size Limit %d Exceeded! Current size is %d", s.volumeSizeLimit, v.ContentSize())
-		}
-		if s.volumeSizeLimit < v.ContentSize()+3*uint64(size) {
-			glog.V(0).Infoln("volume", i, "size", v.ContentSize(), "will exceed limit", s.volumeSizeLimit)
-			if _, _, e := s.SendHeartbeatToMaster(); e != nil {
-				glog.V(0).Infoln("error when reporting size:", e)
-			}
-		}
-		return
-	}
-	glog.V(0).Infoln("volume", i, "not found!")
-	err = fmt.Errorf("Volume %d not found!", i)
-	return
-}
-func (s *Store) Delete(i VolumeId, n *Needle) (uint32, error) {
-	if v := s.findVolume(i); v != nil && !v.readOnly {
-		return v.delete(n)
-	}
-	return 0, nil
-}
-func (s *Store) Read(i VolumeId, n *Needle) (int, error) {
-	if v := s.findVolume(i); v != nil {
-		return v.read(n)
-	}
-	return 0, fmt.Errorf("Volume %v not found!", i)
-}
-func (s *Store) GetVolume(i VolumeId) *Volume {
-	return s.findVolume(i)
-}
-
-func (s *Store) HasVolume(i VolumeId) bool {
-	v := s.findVolume(i)
-	return v != nil
-}

go/storage/store_vacuum.go

@@ -1,44 +0,0 @@
-package storage
-
-import (
-	"fmt"
-	"strconv"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-)
-
-func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) {
-	vid, err := NewVolumeId(volumeIdString)
-	if err != nil {
-		return fmt.Errorf("Volume Id %s is not a valid unsigned integer", volumeIdString), false
-	}
-	garbageThreshold, e := strconv.ParseFloat(garbageThresholdString, 32)
-	if e != nil {
-		return fmt.Errorf("garbageThreshold %s is not a valid float number", garbageThresholdString), false
-	}
-	if v := s.findVolume(vid); v != nil {
-		glog.V(3).Infoln(vid, "garbage level is", v.garbageLevel())
-		return nil, garbageThreshold < v.garbageLevel()
-	}
-	return fmt.Errorf("volume id %d is not found during check compact", vid), false
-}
-func (s *Store) CompactVolume(volumeIdString string) error {
-	vid, err := NewVolumeId(volumeIdString)
-	if err != nil {
-		return fmt.Errorf("Volume Id %s is not a valid unsigned integer", volumeIdString)
-	}
-	if v := s.findVolume(vid); v != nil {
-		return v.Compact()
-	}
-	return fmt.Errorf("volume id %d is not found during compact", vid)
-}
-func (s *Store) CommitCompactVolume(volumeIdString string) error {
-	vid, err := NewVolumeId(volumeIdString)
-	if err != nil {
-		return fmt.Errorf("Volume Id %s is not a valid unsigned integer", volumeIdString)
-	}
-	if v := s.findVolume(vid); v != nil {
-		return v.commitCompact()
-	}
-	return fmt.Errorf("volume id %d is not found during commit compact", vid)
-}

go/storage/volume.go

@@ -1,392 +0,0 @@
-package storage
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"path"
-	"sync"
-	"time"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-)
-
-type Volume struct {
-	Id            VolumeId
-	dir           string
-	Collection    string
-	dataFile      *os.File
-	nm            NeedleMapper
-	needleMapKind NeedleMapType
-	readOnly      bool
-
-	SuperBlock
-
-	accessLock       sync.Mutex
-	lastModifiedTime uint64 //unix time in seconds
-}
-
-func NewVolume(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL) (v *Volume, e error) {
-	v = &Volume{dir: dirname, Collection: collection, Id: id}
-	v.SuperBlock = SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}
-	v.needleMapKind = needleMapKind
-	e = v.load(true, true, needleMapKind)
-	return
-}
-func (v *Volume) String() string {
-	return fmt.Sprintf("Id:%v, dir:%s, Collection:%s, dataFile:%v, nm:%v, readOnly:%v", v.Id, v.dir, v.Collection, v.dataFile, v.nm, v.readOnly)
-}
-
-func loadVolumeWithoutIndex(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType) (v *Volume, e error) {
-	v = &Volume{dir: dirname, Collection: collection, Id: id}
-	v.SuperBlock = SuperBlock{}
-	v.needleMapKind = needleMapKind
-	e = v.load(false, false, needleMapKind)
-	return
-}
-func (v *Volume) FileName() (fileName string) {
-	if v.Collection == "" {
-		fileName = path.Join(v.dir, v.Id.String())
-	} else {
-		fileName = path.Join(v.dir, v.Collection+"_"+v.Id.String())
-	}
-	return
-}
-func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType) error {
-	var e error
-	fileName := v.FileName()
-
-	if exists, canRead, canWrite, modifiedTime := checkFile(fileName + ".dat"); exists {
-		if !canRead {
-			return fmt.Errorf("cannot read Volume Data file %s.dat", fileName)
-		}
-		if canWrite {
-			v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
-			v.lastModifiedTime = uint64(modifiedTime.Unix())
-		} else {
-			glog.V(0).Infoln("opening " + fileName + ".dat in READONLY mode")
-			v.dataFile, e = os.Open(fileName + ".dat")
-			v.readOnly = true
-		}
-	} else {
-		if createDatIfMissing {
-			v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
-		} else {
-			return fmt.Errorf("Volume Data file %s.dat does not exist.", fileName)
-		}
-	}
-
-	if e != nil {
-		if !os.IsPermission(e) {
-			return fmt.Errorf("cannot load Volume Data %s.dat: %v", fileName, e)
-		}
-	}
-
-	if v.ReplicaPlacement == nil {
-		e = v.readSuperBlock()
-	} else {
-		e = v.maybeWriteSuperBlock()
-	}
-	if e == nil && alsoLoadIndex {
-		var indexFile *os.File
-		if v.readOnly {
-			glog.V(1).Infoln("open to read file", fileName+".idx")
-			if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); e != nil {
-				return fmt.Errorf("cannot read Volume Index %s.idx: %v", fileName, e)
-			}
-		} else {
-			glog.V(1).Infoln("open to write file", fileName+".idx")
-			if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); e != nil {
-				return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, e)
-			}
-		}
-		switch needleMapKind {
-		case NeedleMapInMemory:
-			glog.V(0).Infoln("loading index file", fileName+".idx", "readonly", v.readOnly)
-			if v.nm, e = LoadNeedleMap(indexFile); e != nil {
-				glog.V(0).Infof("loading index %s error: %v", fileName+".idx", e)
-			}
-		case NeedleMapLevelDb:
-			glog.V(0).Infoln("loading leveldb file", fileName+".ldb")
-			if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile); e != nil {
-				glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
-			}
-		case NeedleMapBoltDb:
-			glog.V(0).Infoln("loading boltdb file", fileName+".bdb")
-			if v.nm, e = NewBoltDbNeedleMap(fileName+".bdb", indexFile); e != nil {
-				glog.V(0).Infof("loading boltdb %s error: %v", fileName+".bdb", e)
-			}
-		}
-	}
-	return e
-}
-func (v *Volume) Version() Version {
-	return v.SuperBlock.Version()
-}
-func (v *Volume) Size() int64 {
-	stat, e := v.dataFile.Stat()
-	if e == nil {
-		return stat.Size()
-	}
-	glog.V(0).Infof("Failed to read file size %s %v", v.dataFile.Name(), e)
-	return -1
-}
-
-// Close cleanly shuts down this volume
-func (v *Volume) Close() {
-	v.accessLock.Lock()
-	defer v.accessLock.Unlock()
-	v.nm.Close()
-	_ = v.dataFile.Close()
-}
-
-func (v *Volume) NeedToReplicate() bool {
-	return v.ReplicaPlacement.GetCopyCount() > 1
-}
-
-// isFileUnchanged checks whether this needle to write is same as last one.
-// It requires serialized access in the same volume.
-func (v *Volume) isFileUnchanged(n *Needle) bool {
-	nv, ok := v.nm.Get(n.Id)
-	if ok && nv.Offset > 0 {
-		oldNeedle := new(Needle)
-		oldNeedle.Read(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
-		if oldNeedle.Checksum == n.Checksum && bytes.Equal(oldNeedle.Data, n.Data) {
-			n.DataSize = oldNeedle.DataSize
-			return true
-		}
-	}
-	return false
-}
-
-// Destroy removes everything related to this volume
-func (v *Volume) Destroy() (err error) {
-	if v.readOnly {
-		err = fmt.Errorf("%s is read-only", v.dataFile.Name())
-		return
-	}
-	v.Close()
-	err = os.Remove(v.dataFile.Name())
-	if err != nil {
-		return
-	}
-	err = v.nm.Destroy()
-	return
-}
-
-func (v *Volume) write(n *Needle) (size uint32, err error) {
-	glog.V(4).Infof("writing needle %s", NewFileIdFromNeedle(v.Id, n).String())
-	if v.readOnly {
-		err = fmt.Errorf("%s is read-only", v.dataFile.Name())
-		return
-	}
-	v.accessLock.Lock()
-	defer v.accessLock.Unlock()
-	if v.isFileUnchanged(n) {
-		size = n.DataSize
-		glog.V(4).Infof("needle is unchanged!")
-		return
-	}
-	var offset int64
-	if offset, err = v.dataFile.Seek(0, 2); err != nil {
-		glog.V(0).Infof("failed to seek the end of file: %v", err)
-		return
-	}
-
-	//ensure file writing starting from aligned positions
-	if offset%NeedlePaddingSize != 0 {
-		offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize)
-		if offset, err = v.dataFile.Seek(offset, 0); err != nil {
-			glog.V(0).Infof("failed to align in datafile %s: %v", v.dataFile.Name(), err)
-			return
-		}
-	}
-
-	if size, err = n.Append(v.dataFile, v.Version()); err != nil {
-		if e := v.dataFile.Truncate(offset); e != nil {
-			err = fmt.Errorf("%s\ncannot truncate %s: %v", err, v.dataFile.Name(), e)
-		}
-		return
-	}
-	nv, ok := v.nm.Get(n.Id)
-	if !ok || int64(nv.Offset)*NeedlePaddingSize < offset {
-		if err = v.nm.Put(n.Id, uint32(offset/NeedlePaddingSize), n.Size); err != nil {
-			glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err)
-		}
-	}
-	if v.lastModifiedTime < n.LastModified {
-		v.lastModifiedTime = n.LastModified
-	}
-	return
-}
-
-func (v *Volume) delete(n *Needle) (uint32, error) {
-	glog.V(4).Infof("delete needle %s", NewFileIdFromNeedle(v.Id, n).String())
-	if v.readOnly {
-		return 0, fmt.Errorf("%s is read-only", v.dataFile.Name())
-	}
-	v.accessLock.Lock()
-	defer v.accessLock.Unlock()
-	nv, ok := v.nm.Get(n.Id)
-	//fmt.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
-	if ok {
-		size := nv.Size
-		if err := v.nm.Delete(n.Id); err != nil {
-			return size, err
-		}
-		if _, err := v.dataFile.Seek(0, 2); err != nil {
-			return size, err
-		}
-		n.Data = nil
-		_, err := n.Append(v.dataFile, v.Version())
-		return size, err
-	}
-	return 0, nil
-}
-
-func (v *Volume) read(n *Needle) (int, error) {
-	nv, ok := v.nm.Get(n.Id)
-	if !ok || nv.Offset == 0 {
-		return -1, errors.New("Not Found")
-	}
-	bytesRead, err := n.Read(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
-	if err != nil {
-		return bytesRead, err
-	}
-	if !n.HasTtl() {
-		return bytesRead, err
-	}
-	ttlMinutes := n.Ttl.Minutes()
-	if ttlMinutes == 0 {
-		return bytesRead, nil
-	}
-	if !n.HasLastModifiedDate() {
-		return bytesRead, nil
-	}
-	if uint64(time.Now().Unix()) < n.LastModified+uint64(ttlMinutes*60) {
-		return bytesRead, nil
-	}
-	return -1, errors.New("Not Found")
-}
-
-func ScanVolumeFile(dirname string, collection string, id VolumeId,
-	needleMapKind NeedleMapType,
-	visitSuperBlock func(SuperBlock) error,
-	readNeedleBody bool,
-	visitNeedle func(n *Needle, offset int64) error) (err error) {
-	var v *Volume
-	if v, err = loadVolumeWithoutIndex(dirname, collection, id, needleMapKind); err != nil {
-		return fmt.Errorf("Failed to load volume %d: %v", id, err)
-	}
-	if err = visitSuperBlock(v.SuperBlock); err != nil {
-		return fmt.Errorf("Failed to read volume %d super block: %v", id, err)
-	}
-
-	version := v.Version()
-
-	offset := int64(SuperBlockSize)
-	n, rest, e := ReadNeedleHeader(v.dataFile, version, offset)
-	if e != nil {
-		err = fmt.Errorf("cannot read needle header: %v", e)
-		return
-	}
-	for n != nil {
-		if readNeedleBody {
-			if err = n.ReadNeedleBody(v.dataFile, version, offset+int64(NeedleHeaderSize), rest); err != nil {
-				glog.V(0).Infof("cannot read needle body: %v", err)
-				//err = fmt.Errorf("cannot read needle body: %v", err)
-				//return
-			}
-			if n.DataSize >= n.Size {
-				// this should come from a bug reported on #87 and #93
-				// fixed in v0.69
-				// remove this whole "if" clause later, long after 0.69
-				oldRest, oldSize := rest, n.Size
-				padding := NeedlePaddingSize - ((n.Size + NeedleHeaderSize + NeedleChecksumSize) % NeedlePaddingSize)
-				n.Size = 0
-				rest = n.Size + NeedleChecksumSize + padding
-				if rest%NeedlePaddingSize != 0 {
-					rest += (NeedlePaddingSize - rest%NeedlePaddingSize)
-				}
-				glog.V(4).Infof("Adjusting n.Size %d=>0 rest:%d=>%d %+v", oldSize, oldRest, rest, n)
-			}
-		}
-		if err = visitNeedle(n, offset); err != nil {
-			glog.V(0).Infof("visit needle error: %v", err)
-		}
-		offset += int64(NeedleHeaderSize) + int64(rest)
-		glog.V(4).Infof("==> new entry offset %d", offset)
-		if n, rest, err = ReadNeedleHeader(v.dataFile, version, offset); err != nil {
-			if err == io.EOF {
-				return nil
-			}
-			return fmt.Errorf("cannot read needle header: %v", err)
-		}
-		glog.V(4).Infof("new entry needle size:%d rest:%d", n.Size, rest)
-	}
-
-	return
-}
-
-func (v *Volume) ContentSize() uint64 {
-	return v.nm.ContentSize()
-}
-
-func checkFile(filename string) (exists, canRead, canWrite bool, modTime time.Time) {
-	exists = true
-	fi, err := os.Stat(filename)
-	if os.IsNotExist(err) {
-		exists = false
-		return
-	}
-	if fi.Mode()&0400 != 0 {
-		canRead = true
-	}
-	if fi.Mode()&0200 != 0 {
-		canWrite = true
-	}
-	modTime = fi.ModTime()
-	return
-}
-
-// volume is expired if modified time + volume ttl < now
-// except when volume is empty
-// or when the volume does not have a ttl
-// or when volumeSizeLimit is 0 when server just starts
-func (v *Volume) expired(volumeSizeLimit uint64) bool {
-	if volumeSizeLimit == 0 {
-		//skip if we don't know size limit
-		return false
-	}
-	if v.ContentSize() == 0 {
-		return false
-	}
-	if v.Ttl == nil || v.Ttl.Minutes() == 0 {
-		return false
-	}
-	glog.V(0).Infof("now:%v lastModified:%v", time.Now().Unix(), v.lastModifiedTime)
-	livedMinutes := (time.Now().Unix() - int64(v.lastModifiedTime)) / 60
-	glog.V(0).Infof("ttl:%v lived:%v", v.Ttl, livedMinutes)
-	if int64(v.Ttl.Minutes()) < livedMinutes {
-		return true
-	}
-	return false
-}
-
-// wait either maxDelayMinutes or 10% of ttl minutes
-func (v *Volume) exiredLongEnough(maxDelayMinutes uint32) bool {
-	if v.Ttl == nil || v.Ttl.Minutes() == 0 {
-		return false
-	}
-	removalDelay := v.Ttl.Minutes() / 10
-	if removalDelay > maxDelayMinutes {
-		removalDelay = maxDelayMinutes
-	}
-
-	if uint64(v.Ttl.Minutes()+removalDelay)*60+v.lastModifiedTime < uint64(time.Now().Unix()) {
-		return true
-	}
-	return false
-}

go/storage/volume_info.go

@@ -1,44 +0,0 @@
-package storage
-
-import (
-	"fmt"
-	"github.com/chrislusf/weed-fs/go/operation"
-)
-
-type VolumeInfo struct {
-	Id               VolumeId
-	Size             uint64
-	ReplicaPlacement *ReplicaPlacement
-	Ttl              *TTL
-	Collection       string
-	Version          Version
-	FileCount        int
-	DeleteCount      int
-	DeletedByteCount uint64
-	ReadOnly         bool
-}
-
-func NewVolumeInfo(m *operation.VolumeInformationMessage) (vi VolumeInfo, err error) {
-	vi = VolumeInfo{
-		Id:               VolumeId(*m.Id),
-		Size:             *m.Size,
-		Collection:       *m.Collection,
-		FileCount:        int(*m.FileCount),
-		DeleteCount:      int(*m.DeleteCount),
-		DeletedByteCount: *m.DeletedByteCount,
-		ReadOnly:         *m.ReadOnly,
-		Version:          Version(*m.Version),
-	}
-	rp, e := NewReplicaPlacementFromByte(byte(*m.ReplicaPlacement))
-	if e != nil {
-		return vi, e
-	}
-	vi.ReplicaPlacement = rp
-	vi.Ttl = LoadTTLFromUint32(*m.Ttl)
-	return vi, nil
-}
-
-func (vi VolumeInfo) String() string {
-	return fmt.Sprintf("Id:%d, Size:%d, ReplicaPlacement:%s, Collection:%s, Version:%v, FileCount:%d, DeleteCount:%d, DeletedByteCount:%d, ReadOnly:%v",
-		vi.Id, vi.Size, vi.ReplicaPlacement, vi.Collection, vi.Version, vi.FileCount, vi.DeleteCount, vi.DeletedByteCount, vi.ReadOnly)
-}

go/storage/volume_super_block.go

@@ -1,76 +0,0 @@
-package storage
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-)
-
-const (
-	SuperBlockSize = 8
-)
-
-/*
-* Super block currently has 8 bytes allocated for each volume.
-* Byte 0: version, 1 or 2
-* Byte 1: Replica Placement strategy, 000, 001, 002, 010, etc
-* Byte 2 and byte 3: Time to live. See TTL for definition
-* Rest bytes: Reserved
- */
-type SuperBlock struct {
-	version          Version
-	ReplicaPlacement *ReplicaPlacement
-	Ttl              *TTL
-}
-
-func (s *SuperBlock) Version() Version {
-	return s.version
-}
-func (s *SuperBlock) Bytes() []byte {
-	header := make([]byte, SuperBlockSize)
-	header[0] = byte(s.version)
-	header[1] = s.ReplicaPlacement.Byte()
-	s.Ttl.ToBytes(header[2:4])
-	return header
-}
-
-func (v *Volume) maybeWriteSuperBlock() error {
-	stat, e := v.dataFile.Stat()
-	if e != nil {
-		glog.V(0).Infof("failed to stat datafile %s: %v", v.dataFile, e)
-		return e
-	}
-	if stat.Size() == 0 {
-		v.SuperBlock.version = CurrentVersion
-		_, e = v.dataFile.Write(v.SuperBlock.Bytes())
-		if e != nil && os.IsPermission(e) {
-			//read-only, but zero length - recreate it!
-			if v.dataFile, e = os.Create(v.dataFile.Name()); e == nil {
-				if _, e = v.dataFile.Write(v.SuperBlock.Bytes()); e == nil {
-					v.readOnly = false
-				}
-			}
-		}
-	}
-	return e
-}
-func (v *Volume) readSuperBlock() (err error) {
-	if _, err = v.dataFile.Seek(0, 0); err != nil {
-		return fmt.Errorf("cannot seek to the beginning of %s: %v", v.dataFile.Name(), err)
-	}
-	header := make([]byte, SuperBlockSize)
-	if _, e := v.dataFile.Read(header); e != nil {
-		return fmt.Errorf("cannot read volume %d super block: %v", v.Id, e)
-	}
-	v.SuperBlock, err = ParseSuperBlock(header)
-	return err
-}
-func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) {
-	superBlock.version = Version(header[0])
-	if superBlock.ReplicaPlacement, err = NewReplicaPlacementFromByte(header[1]); err != nil {
-		err = fmt.Errorf("cannot read replica type: %s", err.Error())
-	}
-	superBlock.Ttl = LoadTTLFromBytes(header[2:4])
-	return
-}

go/storage/volume_vacuum.go

@@ -1,90 +0,0 @@
-package storage
-
-import (
-	"fmt"
-	"os"
-	"time"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-)
-
-func (v *Volume) garbageLevel() float64 {
-	return float64(v.nm.DeletedSize()) / float64(v.ContentSize())
-}
-
-func (v *Volume) Compact() error {
-	glog.V(3).Infof("Compacting ...")
-	//no need to lock for copy on write
-	//v.accessLock.Lock()
-	//defer v.accessLock.Unlock()
-	//glog.V(3).Infof("Got Compaction lock...")
-
-	filePath := v.FileName()
-	glog.V(3).Infof("creating copies for volume %d ...", v.Id)
-	return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx")
-}
-func (v *Volume) commitCompact() error {
-	glog.V(3).Infof("Committing vacuuming...")
-	v.accessLock.Lock()
-	defer v.accessLock.Unlock()
-	glog.V(3).Infof("Got Committing lock...")
-	_ = v.dataFile.Close()
-	var e error
-	if e = os.Rename(v.FileName()+".cpd", v.FileName()+".dat"); e != nil {
-		return e
-	}
-	if e = os.Rename(v.FileName()+".cpx", v.FileName()+".idx"); e != nil {
-		return e
-	}
-	//glog.V(3).Infof("Pretending to be vacuuming...")
-	//time.Sleep(20 * time.Second)
-	if e = v.load(true, false, v.needleMapKind); e != nil {
-		return e
-	}
-	return nil
-}
-
-func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) {
-	var (
-		dst, idx *os.File
-	)
-	if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
-		return
-	}
-	defer dst.Close()
-
-	if idx, err = os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
-		return
-	}
-	defer idx.Close()
-
-	nm := NewNeedleMap(idx)
-	new_offset := int64(SuperBlockSize)
-
-	now := uint64(time.Now().Unix())
-
-	err = ScanVolumeFile(v.dir, v.Collection, v.Id, v.needleMapKind,
-		func(superBlock SuperBlock) error {
-			_, err = dst.Write(superBlock.Bytes())
-			return err
-		}, true, func(n *Needle, offset int64) error {
-			if n.HasTtl() && now >= n.LastModified+uint64(v.Ttl.Minutes()*60) {
-				return nil
-			}
-			nv, ok := v.nm.Get(n.Id)
-			glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv)
-			if ok && int64(nv.Offset)*NeedlePaddingSize == offset && nv.Size > 0 {
-				if err = nm.Put(n.Id, uint32(new_offset/NeedlePaddingSize), n.Size); err != nil {
-					return fmt.Errorf("cannot put needle: %s", err)
-				}
-				if _, err = n.Append(dst, v.Version()); err != nil {
-					return fmt.Errorf("cannot append needle: %s", err)
-				}
-				new_offset += n.DiskSize()
-				glog.V(3).Infoln("saving key", n.Id, "volume offset", offset, "=>", new_offset, "data_size", n.Size)
-			}
-			return nil
-		})
-
-	return
-}

go/tools/read_index.go

@@ -1,28 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"os"
-
-	"github.com/chrislusf/weed-fs/go/storage"
-)
-
-var (
-	indexFileName = flag.String("file", "", ".idx file to analyze")
-)
-
-func main() {
-	flag.Parse()
-	indexFile, err := os.OpenFile(*indexFileName, os.O_RDONLY, 0644)
-	if err != nil {
-		log.Fatalf("Create Volume Index [ERROR] %s\n", err)
-	}
-	defer indexFile.Close()
-
-	storage.WalkIndexFile(indexFile, func(key uint64, offset, size uint32) error {
-		fmt.Printf("key %d, offset %d, size %d, nextOffset %d\n", key, offset*8, size, offset*8+size)
-		return nil
-	})
-}

go/topology/allocate_volume.go

@@ -1,35 +0,0 @@
-package topology
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net/url"
-
-	"github.com/chrislusf/weed-fs/go/storage"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-type AllocateVolumeResult struct {
-	Error string
-}
-
-func AllocateVolume(dn *DataNode, vid storage.VolumeId, option *VolumeGrowOption) error {
-	values := make(url.Values)
-	values.Add("volume", vid.String())
-	values.Add("collection", option.Collection)
-	values.Add("replication", option.ReplicaPlacement.String())
-	values.Add("ttl", option.Ttl.String())
-	jsonBlob, err := util.Post("http://"+dn.Url()+"/admin/assign_volume", values)
-	if err != nil {
-		return err
-	}
-	var ret AllocateVolumeResult
-	if err := json.Unmarshal(jsonBlob, &ret); err != nil {
-		return fmt.Errorf("Invalid JSON result for %s: %s", "/admin/assign_volum", string(jsonBlob))
-	}
-	if ret.Error != "" {
-		return errors.New(ret.Error)
-	}
-	return nil
-}

go/topology/configuration_test.go

@@ -1,42 +0,0 @@
-package topology
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestLoadConfiguration(t *testing.T) {
-
-	confContent := `
-
-<?xml version="1.0" encoding="UTF-8" ?>
-<Configuration>
-  <Topology>
-    <DataCenter name="dc1">
-      <Rack name="rack1">
-        <Ip>192.168.1.1</Ip>
-      </Rack>
-    </DataCenter>
-    <DataCenter name="dc2">
-      <Rack name="rack1">
-        <Ip>192.168.1.2</Ip>
-      </Rack>
-      <Rack name="rack2">
-        <Ip>192.168.1.3</Ip>
-        <Ip>192.168.1.4</Ip>
-      </Rack>
-    </DataCenter>
-  </Topology>
-</Configuration>
-`
-	c, err := NewConfiguration([]byte(confContent))
-
-	fmt.Printf("%s\n", c)
-	if err != nil {
-		t.Fatalf("unmarshal error:%v", err)
-	}
-
-	if len(c.Topo.DataCenters) <= 0 || c.Topo.DataCenters[0].Name != "dc1" {
-		t.Fatalf("unmarshal error:%s", c)
-	}
-}

go/topology/data_node.go

@@ -1,100 +0,0 @@
-package topology
-
-import (
-	"fmt"
-	"strconv"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/storage"
-)
-
-type DataNode struct {
-	NodeImpl
-	volumes   map[storage.VolumeId]storage.VolumeInfo
-	Ip        string
-	Port      int
-	PublicUrl string
-	LastSeen  int64 // unix time in seconds
-	Dead      bool
-}
-
-func NewDataNode(id string) *DataNode {
-	s := &DataNode{}
-	s.id = NodeId(id)
-	s.nodeType = "DataNode"
-	s.volumes = make(map[storage.VolumeId]storage.VolumeInfo)
-	s.NodeImpl.value = s
-	return s
-}
-
-func (dn *DataNode) String() string {
-	return fmt.Sprintf("Node:%s, volumes:%v, Ip:%s, Port:%d, PublicUrl:%s, Dead:%v", dn.NodeImpl.String(), dn.volumes, dn.Ip, dn.Port, dn.PublicUrl, dn.Dead)
-}
-
-func (dn *DataNode) AddOrUpdateVolume(v storage.VolumeInfo) {
-	if _, ok := dn.volumes[v.Id]; !ok {
-		dn.volumes[v.Id] = v
-		dn.UpAdjustVolumeCountDelta(1)
-		if !v.ReadOnly {
-			dn.UpAdjustActiveVolumeCountDelta(1)
-		}
-		dn.UpAdjustMaxVolumeId(v.Id)
-	} else {
-		dn.volumes[v.Id] = v
-	}
-}
-
-func (dn *DataNode) UpdateVolumes(actualVolumes []storage.VolumeInfo) (deletedVolumes []storage.VolumeInfo) {
-	actualVolumeMap := make(map[storage.VolumeId]storage.VolumeInfo)
-	for _, v := range actualVolumes {
-		actualVolumeMap[v.Id] = v
-	}
-	for vid, v := range dn.volumes {
-		if _, ok := actualVolumeMap[vid]; !ok {
-			glog.V(0).Infoln("Deleting volume id:", vid)
-			delete(dn.volumes, vid)
-			deletedVolumes = append(deletedVolumes, v)
-			dn.UpAdjustVolumeCountDelta(-1)
-			dn.UpAdjustActiveVolumeCountDelta(-1)
-		}
-	} //TODO: adjust max volume id, if need to reclaim volume ids
-	for _, v := range actualVolumes {
-		dn.AddOrUpdateVolume(v)
-	}
-	return
-}
-
-func (dn *DataNode) GetDataCenter() *DataCenter {
-	return dn.Parent().Parent().(*NodeImpl).value.(*DataCenter)
-}
-
-func (dn *DataNode) GetRack() *Rack {
-	return dn.Parent().(*NodeImpl).value.(*Rack)
-}
-
-func (dn *DataNode) GetTopology() *Topology {
-	p := dn.Parent()
-	for p.Parent() != nil {
-		p = p.Parent()
-	}
-	t := p.(*Topology)
-	return t
-}
-
-func (dn *DataNode) MatchLocation(ip string, port int) bool {
-	return dn.Ip == ip && dn.Port == port
-}
-
-func (dn *DataNode) Url() string {
-	return dn.Ip + ":" + strconv.Itoa(dn.Port)
-}
-
-func (dn *DataNode) ToMap() interface{} {
-	ret := make(map[string]interface{})
-	ret["Url"] = dn.Url()
-	ret["Volumes"] = dn.GetVolumeCount()
-	ret["Max"] = dn.GetMaxVolumeCount()
-	ret["Free"] = dn.FreeSpace()
-	ret["PublicUrl"] = dn.PublicUrl
-	return ret
-}

go/topology/node.go

@@ -1,255 +0,0 @@
-package topology
-
-import (
-	"errors"
-	"math/rand"
-	"strings"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/storage"
-)
-
-type NodeId string
-type Node interface {
-	Id() NodeId
-	String() string
-	FreeSpace() int
-	ReserveOneVolume(r int) (*DataNode, error)
-	UpAdjustMaxVolumeCountDelta(maxVolumeCountDelta int)
-	UpAdjustVolumeCountDelta(volumeCountDelta int)
-	UpAdjustActiveVolumeCountDelta(activeVolumeCountDelta int)
-	UpAdjustMaxVolumeId(vid storage.VolumeId)
-
-	GetVolumeCount() int
-	GetActiveVolumeCount() int
-	GetMaxVolumeCount() int
-	GetMaxVolumeId() storage.VolumeId
-	SetParent(Node)
-	LinkChildNode(node Node)
-	UnlinkChildNode(nodeId NodeId)
-	CollectDeadNodeAndFullVolumes(freshThreshHold int64, volumeSizeLimit uint64)
-
-	IsDataNode() bool
-	IsRack() bool
-	IsDataCenter() bool
-	Children() map[NodeId]Node
-	Parent() Node
-
-	GetValue() interface{} //get reference to the topology,dc,rack,datanode
-}
-type NodeImpl struct {
-	id                NodeId
-	volumeCount       int
-	activeVolumeCount int
-	maxVolumeCount    int
-	parent            Node
-	children          map[NodeId]Node
-	maxVolumeId       storage.VolumeId
-
-	//for rack, data center, topology
-	nodeType string
-	value    interface{}
-}
-
-// the first node must satisfy filterFirstNodeFn(), the rest nodes must have one free slot
-func (n *NodeImpl) RandomlyPickNodes(numberOfNodes int, filterFirstNodeFn func(dn Node) error) (firstNode Node, restNodes []Node, err error) {
-	candidates := make([]Node, 0, len(n.children))
-	var errs []string
-	for _, node := range n.children {
-		if err := filterFirstNodeFn(node); err == nil {
-			candidates = append(candidates, node)
-		} else {
-			errs = append(errs, string(node.Id())+":"+err.Error())
-		}
-	}
-	if len(candidates) == 0 {
-		return nil, nil, errors.New("No matching data node found! \n" + strings.Join(errs, "\n"))
-	}
-	firstNode = candidates[rand.Intn(len(candidates))]
-	glog.V(2).Infoln(n.Id(), "picked main node:", firstNode.Id())
-
-	restNodes = make([]Node, numberOfNodes-1)
-	candidates = candidates[:0]
-	for _, node := range n.children {
-		if node.Id() == firstNode.Id() {
-			continue
-		}
-		if node.FreeSpace() <= 0 {
-			continue
-		}
-		glog.V(2).Infoln("select rest node candidate:", node.Id())
-		candidates = append(candidates, node)
-	}
-	glog.V(2).Infoln(n.Id(), "picking", numberOfNodes-1, "from rest", len(candidates), "node candidates")
-	ret := len(restNodes) == 0
-	for k, node := range candidates {
-		if k < len(restNodes) {
-			restNodes[k] = node
-			if k == len(restNodes)-1 {
-				ret = true
-			}
-		} else {
-			r := rand.Intn(k + 1)
-			if r < len(restNodes) {
-				restNodes[r] = node
-			}
-		}
-	}
-	if !ret {
-		glog.V(2).Infoln(n.Id(), "failed to pick", numberOfNodes-1, "from rest", len(candidates), "node candidates")
-		err = errors.New("Not enough data node found!")
-	}
-	return
-}
-
-func (n *NodeImpl) IsDataNode() bool {
-	return n.nodeType == "DataNode"
-}
-func (n *NodeImpl) IsRack() bool {
-	return n.nodeType == "Rack"
-}
-func (n *NodeImpl) IsDataCenter() bool {
-	return n.nodeType == "DataCenter"
-}
-func (n *NodeImpl) String() string {
-	if n.parent != nil {
-		return n.parent.String() + ":" + string(n.id)
-	}
-	return string(n.id)
-}
-func (n *NodeImpl) Id() NodeId {
-	return n.id
-}
-func (n *NodeImpl) FreeSpace() int {
-	return n.maxVolumeCount - n.volumeCount
-}
-func (n *NodeImpl) SetParent(node Node) {
-	n.parent = node
-}
-func (n *NodeImpl) Children() map[NodeId]Node {
-	return n.children
-}
-func (n *NodeImpl) Parent() Node {
-	return n.parent
-}
-func (n *NodeImpl) GetValue() interface{} {
-	return n.value
-}
-func (n *NodeImpl) ReserveOneVolume(r int) (assignedNode *DataNode, err error) {
-	for _, node := range n.children {
-		freeSpace := node.FreeSpace()
-		// fmt.Println("r =", r, ", node =", node, ", freeSpace =", freeSpace)
-		if freeSpace <= 0 {
-			continue
-		}
-		if r >= freeSpace {
-			r -= freeSpace
-		} else {
-			if node.IsDataNode() && node.FreeSpace() > 0 {
-				// fmt.Println("vid =", vid, " assigned to node =", node, ", freeSpace =", node.FreeSpace())
-				return node.(*DataNode), nil
-			}
-			assignedNode, err = node.ReserveOneVolume(r)
-			if err != nil {
-				return
-			}
-		}
-	}
-	return
-}
-
-func (n *NodeImpl) UpAdjustMaxVolumeCountDelta(maxVolumeCountDelta int) { //can be negative
-	n.maxVolumeCount += maxVolumeCountDelta
-	if n.parent != nil {
-		n.parent.UpAdjustMaxVolumeCountDelta(maxVolumeCountDelta)
-	}
-}
-func (n *NodeImpl) UpAdjustVolumeCountDelta(volumeCountDelta int) { //can be negative
-	n.volumeCount += volumeCountDelta
-	if n.parent != nil {
-		n.parent.UpAdjustVolumeCountDelta(volumeCountDelta)
-	}
-}
-func (n *NodeImpl) UpAdjustActiveVolumeCountDelta(activeVolumeCountDelta int) { //can be negative
-	n.activeVolumeCount += activeVolumeCountDelta
-	if n.parent != nil {
-		n.parent.UpAdjustActiveVolumeCountDelta(activeVolumeCountDelta)
-	}
-}
-func (n *NodeImpl) UpAdjustMaxVolumeId(vid storage.VolumeId) { //can be negative
-	if n.maxVolumeId < vid {
-		n.maxVolumeId = vid
-		if n.parent != nil {
-			n.parent.UpAdjustMaxVolumeId(vid)
-		}
-	}
-}
-func (n *NodeImpl) GetMaxVolumeId() storage.VolumeId {
-	return n.maxVolumeId
-}
-func (n *NodeImpl) GetVolumeCount() int {
-	return n.volumeCount
-}
-func (n *NodeImpl) GetActiveVolumeCount() int {
-	return n.activeVolumeCount
-}
-func (n *NodeImpl) GetMaxVolumeCount() int {
-	return n.maxVolumeCount
-}
-
-func (n *NodeImpl) LinkChildNode(node Node) {
-	if n.children[node.Id()] == nil {
-		n.children[node.Id()] = node
-		n.UpAdjustMaxVolumeCountDelta(node.GetMaxVolumeCount())
-		n.UpAdjustMaxVolumeId(node.GetMaxVolumeId())
-		n.UpAdjustVolumeCountDelta(node.GetVolumeCount())
-		n.UpAdjustActiveVolumeCountDelta(node.GetActiveVolumeCount())
-		node.SetParent(n)
-		glog.V(0).Infoln(n, "adds child", node.Id())
-	}
-}
-
-func (n *NodeImpl) UnlinkChildNode(nodeId NodeId) {
-	node := n.children[nodeId]
-	node.SetParent(nil)
-	if node != nil {
-		delete(n.children, node.Id())
-		n.UpAdjustVolumeCountDelta(-node.GetVolumeCount())
-		n.UpAdjustActiveVolumeCountDelta(-node.GetActiveVolumeCount())
-		n.UpAdjustMaxVolumeCountDelta(-node.GetMaxVolumeCount())
-		glog.V(0).Infoln(n, "removes", node, "volumeCount =", n.activeVolumeCount)
-	}
-}
-
-func (n *NodeImpl) CollectDeadNodeAndFullVolumes(freshThreshHold int64, volumeSizeLimit uint64) {
-	if n.IsRack() {
-		for _, c := range n.Children() {
-			dn := c.(*DataNode) //can not cast n to DataNode
-			if dn.LastSeen < freshThreshHold {
-				if !dn.Dead {
-					dn.Dead = true
-					n.GetTopology().chanDeadDataNodes <- dn
-				}
-			}
-			for _, v := range dn.volumes {
-				if uint64(v.Size) >= volumeSizeLimit {
-					//fmt.Println("volume",v.Id,"size",v.Size,">",volumeSizeLimit)
-					n.GetTopology().chanFullVolumes <- v
-				}
-			}
-		}
-	} else {
-		for _, c := range n.Children() {
-			c.CollectDeadNodeAndFullVolumes(freshThreshHold, volumeSizeLimit)
-		}
-	}
-}
-
-func (n *NodeImpl) GetTopology() *Topology {
-	var p Node
-	p = n
-	for p.Parent() != nil {
-		p = p.Parent()
-	}
-	return p.GetValue().(*Topology)
-}

go/topology/store_replicate.go

@@ -1,103 +0,0 @@
-package topology
-
-import (
-	"bytes"
-	"net/http"
-	"strconv"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/operation"
-	"github.com/chrislusf/weed-fs/go/security"
-	"github.com/chrislusf/weed-fs/go/storage"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-func ReplicatedWrite(masterNode string, s *storage.Store,
-	volumeId storage.VolumeId, needle *storage.Needle,
-	r *http.Request) (size uint32, errorStatus string) {
-
-	//check JWT
-	jwt := security.GetJwt(r)
-
-	ret, err := s.Write(volumeId, needle)
-	needToReplicate := !s.HasVolume(volumeId)
-	if err != nil {
-		errorStatus = "Failed to write to local disk (" + err.Error() + ")"
-	} else if ret > 0 {
-		needToReplicate = needToReplicate || s.GetVolume(volumeId).NeedToReplicate()
-	} else {
-		errorStatus = "Failed to write to local disk"
-	}
-	if !needToReplicate && ret > 0 {
-		needToReplicate = s.GetVolume(volumeId).NeedToReplicate()
-	}
-	if needToReplicate { //send to other replica locations
-		if r.FormValue("type") != "replicate" {
-			if !distributedOperation(masterNode, s, volumeId, func(location operation.Location) bool {
-				_, err := operation.Upload(
-					"http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified, 10),
-					string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime),
-					jwt)
-				return err == nil
-			}) {
-				ret = 0
-				errorStatus = "Failed to write to replicas for volume " + volumeId.String()
-			}
-		}
-	}
-	size = ret
-	return
-}
-
-func ReplicatedDelete(masterNode string, store *storage.Store,
-	volumeId storage.VolumeId, n *storage.Needle,
-	r *http.Request) (ret uint32) {
-
-	//check JWT
-	jwt := security.GetJwt(r)
-
-	ret, err := store.Delete(volumeId, n)
-	if err != nil {
-		glog.V(0).Infoln("delete error:", err)
-		return
-	}
-
-	needToReplicate := !store.HasVolume(volumeId)
-	if !needToReplicate && ret > 0 {
-		needToReplicate = store.GetVolume(volumeId).NeedToReplicate()
-	}
-	if needToReplicate { //send to other replica locations
-		if r.FormValue("type") != "replicate" {
-			if !distributedOperation(masterNode, store, volumeId, func(location operation.Location) bool {
-				return nil == util.Delete("http://"+location.Url+r.URL.Path+"?type=replicate", jwt)
-			}) {
-				ret = 0
-			}
-		}
-	}
-	return
-}
-
-func distributedOperation(masterNode string, store *storage.Store, volumeId storage.VolumeId, op func(location operation.Location) bool) bool {
-	if lookupResult, lookupErr := operation.Lookup(masterNode, volumeId.String()); lookupErr == nil {
-		length := 0
-		selfUrl := (store.Ip + ":" + strconv.Itoa(store.Port))
-		results := make(chan bool)
-		for _, location := range lookupResult.Locations {
-			if location.Url != selfUrl {
-				length++
-				go func(location operation.Location, results chan bool) {
-					results <- op(location)
-				}(location, results)
-			}
-		}
-		ret := true
-		for i := 0; i < length; i++ {
-			ret = ret && <-results
-		}
-		return ret
-	} else {
-		glog.V(0).Infoln("Failed to lookup for", volumeId, lookupErr.Error())
-	}
-	return false
-}

go/topology/topo_test.go

@@ -1,17 +0,0 @@
-package topology
-
-import (
-	"testing"
-)
-
-func TestRemoveDataCenter(t *testing.T) {
-	topo := setup(topologyLayout)
-	topo.UnlinkChildNode(NodeId("dc2"))
-	if topo.GetActiveVolumeCount() != 15 {
-		t.Fail()
-	}
-	topo.UnlinkChildNode(NodeId("dc3"))
-	if topo.GetActiveVolumeCount() != 12 {
-		t.Fail()
-	}
-}

go/topology/topology.go

@@ -1,189 +0,0 @@
-package topology
-
-import (
-	"errors"
-	"io/ioutil"
-	"math/rand"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/operation"
-	"github.com/chrislusf/weed-fs/go/sequence"
-	"github.com/chrislusf/weed-fs/go/storage"
-	"github.com/chrislusf/weed-fs/go/util"
-	"github.com/goraft/raft"
-)
-
-type Topology struct {
-	NodeImpl
-
-	collectionMap *util.ConcurrentReadMap
-
-	pulse int64
-
-	volumeSizeLimit uint64
-
-	Sequence sequence.Sequencer
-
-	chanDeadDataNodes      chan *DataNode
-	chanRecoveredDataNodes chan *DataNode
-	chanFullVolumes        chan storage.VolumeInfo
-
-	configuration *Configuration
-
-	RaftServer raft.Server
-}
-
-func NewTopology(id string, confFile string, seq sequence.Sequencer, volumeSizeLimit uint64, pulse int) (*Topology, error) {
-	t := &Topology{}
-	t.id = NodeId(id)
-	t.nodeType = "Topology"
-	t.NodeImpl.value = t
-	t.children = make(map[NodeId]Node)
-	t.collectionMap = util.NewConcurrentReadMap()
-	t.pulse = int64(pulse)
-	t.volumeSizeLimit = volumeSizeLimit
-
-	t.Sequence = seq
-
-	t.chanDeadDataNodes = make(chan *DataNode)
-	t.chanRecoveredDataNodes = make(chan *DataNode)
-	t.chanFullVolumes = make(chan storage.VolumeInfo)
-
-	err := t.loadConfiguration(confFile)
-
-	return t, err
-}
-
-func (t *Topology) IsLeader() bool {
-	if leader, e := t.Leader(); e == nil {
-		return leader == t.RaftServer.Name()
-	}
-	return false
-}
-
-func (t *Topology) Leader() (string, error) {
-	l := ""
-	if t.RaftServer != nil {
-		l = t.RaftServer.Leader()
-	} else {
-		return "", errors.New("Raft Server not ready yet!")
-	}
-
-	if l == "" {
-		// We are a single node cluster, we are the leader
-		return t.RaftServer.Name(), errors.New("Raft Server not initialized!")
-	}
-
-	return l, nil
-}
-
-func (t *Topology) loadConfiguration(configurationFile string) error {
-	b, e := ioutil.ReadFile(configurationFile)
-	if e == nil {
-		t.configuration, e = NewConfiguration(b)
-		return e
-	}
-	glog.V(0).Infoln("Using default configurations.")
-	return nil
-}
-
-func (t *Topology) Lookup(collection string, vid storage.VolumeId) []*DataNode {
-	//maybe an issue if lots of collections?
-	if collection == "" {
-		for _, c := range t.collectionMap.Items {
-			if list := c.(*Collection).Lookup(vid); list != nil {
-				return list
-			}
-		}
-	} else {
-		if c, ok := t.collectionMap.Items[collection]; ok {
-			return c.(*Collection).Lookup(vid)
-		}
-	}
-	return nil
-}
-
-func (t *Topology) NextVolumeId() storage.VolumeId {
-	vid := t.GetMaxVolumeId()
-	next := vid.Next()
-	go t.RaftServer.Do(NewMaxVolumeIdCommand(next))
-	return next
-}
-
-func (t *Topology) HasWritableVolume(option *VolumeGrowOption) bool {
-	vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl)
-	return vl.GetActiveVolumeCount(option) > 0
-}
-
-func (t *Topology) PickForWrite(count uint64, option *VolumeGrowOption) (string, uint64, *DataNode, error) {
-	vid, count, datanodes, err := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl).PickForWrite(count, option)
-	if err != nil || datanodes.Length() == 0 {
-		return "", 0, nil, errors.New("No writable volumes available!")
-	}
-	fileId, count := t.Sequence.NextFileId(count)
-	return storage.NewFileId(*vid, fileId, rand.Uint32()).String(), count, datanodes.Head(), nil
-}
-
-func (t *Topology) GetVolumeLayout(collectionName string, rp *storage.ReplicaPlacement, ttl *storage.TTL) *VolumeLayout {
-	return t.collectionMap.Get(collectionName, func() interface{} {
-		return NewCollection(collectionName, t.volumeSizeLimit)
-	}).(*Collection).GetOrCreateVolumeLayout(rp, ttl)
-}
-
-func (t *Topology) GetCollection(collectionName string) (*Collection, bool) {
-	c, hasCollection := t.collectionMap.Items[collectionName]
-	return c.(*Collection), hasCollection
-}
-
-func (t *Topology) DeleteCollection(collectionName string) {
-	delete(t.collectionMap.Items, collectionName)
-}
-
-func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
-	t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl).RegisterVolume(&v, dn)
-}
-func (t *Topology) UnRegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
-	glog.Infof("removing volume info:%+v", v)
-	t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl).UnRegisterVolume(&v, dn)
-}
-
-func (t *Topology) ProcessJoinMessage(joinMessage *operation.JoinMessage) {
-	t.Sequence.SetMax(*joinMessage.MaxFileKey)
-	dcName, rackName := t.configuration.Locate(*joinMessage.Ip, *joinMessage.DataCenter, *joinMessage.Rack)
-	dc := t.GetOrCreateDataCenter(dcName)
-	rack := dc.GetOrCreateRack(rackName)
-	dn := rack.FindDataNode(*joinMessage.Ip, int(*joinMessage.Port))
-	if *joinMessage.IsInit && dn != nil {
-		t.UnRegisterDataNode(dn)
-	}
-	dn = rack.GetOrCreateDataNode(*joinMessage.Ip,
-		int(*joinMessage.Port), *joinMessage.PublicUrl,
-		int(*joinMessage.MaxVolumeCount))
-	var volumeInfos []storage.VolumeInfo
-	for _, v := range joinMessage.Volumes {
-		if vi, err := storage.NewVolumeInfo(v); err == nil {
-			volumeInfos = append(volumeInfos, vi)
-		} else {
-			glog.V(0).Infoln("Fail to convert joined volume information:", err.Error())
-		}
-	}
-	deletedVolumes := dn.UpdateVolumes(volumeInfos)
-	for _, v := range volumeInfos {
-		t.RegisterVolumeLayout(v, dn)
-	}
-	for _, v := range deletedVolumes {
-		t.UnRegisterVolumeLayout(v, dn)
-	}
-}
-
-func (t *Topology) GetOrCreateDataCenter(dcName string) *DataCenter {
-	for _, c := range t.Children() {
-		dc := c.(*DataCenter)
-		if string(dc.Id()) == dcName {
-			return dc
-		}
-	}
-	dc := NewDataCenter(dcName)
-	t.LinkChildNode(dc)
-	return dc
-}

go/topology/topology_map.go

@@ -1,53 +0,0 @@
-package topology
-
-func (t *Topology) ToMap() interface{} {
-	m := make(map[string]interface{})
-	m["Max"] = t.GetMaxVolumeCount()
-	m["Free"] = t.FreeSpace()
-	var dcs []interface{}
-	for _, c := range t.Children() {
-		dc := c.(*DataCenter)
-		dcs = append(dcs, dc.ToMap())
-	}
-	m["DataCenters"] = dcs
-	var layouts []interface{}
-	for _, col := range t.collectionMap.Items {
-		c := col.(*Collection)
-		for _, layout := range c.storageType2VolumeLayout.Items {
-			if layout != nil {
-				tmp := layout.(*VolumeLayout).ToMap()
-				tmp["collection"] = c.Name
-				layouts = append(layouts, tmp)
-			}
-		}
-	}
-	m["layouts"] = layouts
-	return m
-}
-
-func (t *Topology) ToVolumeMap() interface{} {
-	m := make(map[string]interface{})
-	m["Max"] = t.GetMaxVolumeCount()
-	m["Free"] = t.FreeSpace()
-	dcs := make(map[NodeId]interface{})
-	for _, c := range t.Children() {
-		dc := c.(*DataCenter)
-		racks := make(map[NodeId]interface{})
-		for _, r := range dc.Children() {
-			rack := r.(*Rack)
-			dataNodes := make(map[NodeId]interface{})
-			for _, d := range rack.Children() {
-				dn := d.(*DataNode)
-				var volumes []interface{}
-				for _, v := range dn.volumes {
-					volumes = append(volumes, v)
-				}
-				dataNodes[d.Id()] = volumes
-			}
-			racks[r.Id()] = dataNodes
-		}
-		dcs[dc.Id()] = racks
-	}
-	m["DataCenters"] = dcs
-	return m
-}

go/topology/topology_vacuum.go

@@ -1,155 +0,0 @@
-package topology
-
-import (
-	"encoding/json"
-	"errors"
-	"net/url"
-	"time"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/storage"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList, garbageThreshold string) bool {
-	ch := make(chan bool, locationlist.Length())
-	for index, dn := range locationlist.list {
-		go func(index int, url string, vid storage.VolumeId) {
-			//glog.V(0).Infoln(index, "Check vacuuming", vid, "on", dn.Url())
-			if e, ret := vacuumVolume_Check(url, vid, garbageThreshold); e != nil {
-				//glog.V(0).Infoln(index, "Error when checking vacuuming", vid, "on", url, e)
-				ch <- false
-			} else {
-				//glog.V(0).Infoln(index, "Checked vacuuming", vid, "on", url, "needVacuum", ret)
-				ch <- ret
-			}
-		}(index, dn.Url(), vid)
-	}
-	isCheckSuccess := true
-	for _ = range locationlist.list {
-		select {
-		case canVacuum := <-ch:
-			isCheckSuccess = isCheckSuccess && canVacuum
-		case <-time.After(30 * time.Minute):
-			isCheckSuccess = false
-			break
-		}
-	}
-	return isCheckSuccess
-}
-func batchVacuumVolumeCompact(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList) bool {
-	vl.removeFromWritable(vid)
-	ch := make(chan bool, locationlist.Length())
-	for index, dn := range locationlist.list {
-		go func(index int, url string, vid storage.VolumeId) {
-			glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
-			if e := vacuumVolume_Compact(url, vid); e != nil {
-				glog.V(0).Infoln(index, "Error when vacuuming", vid, "on", url, e)
-				ch <- false
-			} else {
-				glog.V(0).Infoln(index, "Complete vacuuming", vid, "on", url)
-				ch <- true
-			}
-		}(index, dn.Url(), vid)
-	}
-	isVacuumSuccess := true
-	for _ = range locationlist.list {
-		select {
-		case _ = <-ch:
-		case <-time.After(30 * time.Minute):
-			isVacuumSuccess = false
-			break
-		}
-	}
-	return isVacuumSuccess
-}
-func batchVacuumVolumeCommit(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList) bool {
-	isCommitSuccess := true
-	for _, dn := range locationlist.list {
-		glog.V(0).Infoln("Start Commiting vacuum", vid, "on", dn.Url())
-		if e := vacuumVolume_Commit(dn.Url(), vid); e != nil {
-			glog.V(0).Infoln("Error when committing vacuum", vid, "on", dn.Url(), e)
-			isCommitSuccess = false
-		} else {
-			glog.V(0).Infoln("Complete Commiting vacuum", vid, "on", dn.Url())
-		}
-		if isCommitSuccess {
-			vl.SetVolumeAvailable(dn, vid)
-		}
-	}
-	return isCommitSuccess
-}
-func (t *Topology) Vacuum(garbageThreshold string) int {
-	for _, col := range t.collectionMap.Items {
-		c := col.(*Collection)
-		for _, vl := range c.storageType2VolumeLayout.Items {
-			if vl != nil {
-				volumeLayout := vl.(*VolumeLayout)
-				for vid, locationlist := range volumeLayout.vid2location {
-					if batchVacuumVolumeCheck(volumeLayout, vid, locationlist, garbageThreshold) {
-						if batchVacuumVolumeCompact(volumeLayout, vid, locationlist) {
-							batchVacuumVolumeCommit(volumeLayout, vid, locationlist)
-						}
-					}
-				}
-			}
-		}
-	}
-	return 0
-}
-
-type VacuumVolumeResult struct {
-	Result bool
-	Error  string
-}
-
-func vacuumVolume_Check(urlLocation string, vid storage.VolumeId, garbageThreshold string) (error, bool) {
-	values := make(url.Values)
-	values.Add("volume", vid.String())
-	values.Add("garbageThreshold", garbageThreshold)
-	jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum_volume_check", values)
-	if err != nil {
-		glog.V(0).Infoln("parameters:", values)
-		return err, false
-	}
-	var ret VacuumVolumeResult
-	if err := json.Unmarshal(jsonBlob, &ret); err != nil {
-		return err, false
-	}
-	if ret.Error != "" {
-		return errors.New(ret.Error), false
-	}
-	return nil, ret.Result
-}
-func vacuumVolume_Compact(urlLocation string, vid storage.VolumeId) error {
-	values := make(url.Values)
-	values.Add("volume", vid.String())
-	jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum_volume_compact", values)
-	if err != nil {
-		return err
-	}
-	var ret VacuumVolumeResult
-	if err := json.Unmarshal(jsonBlob, &ret); err != nil {
-		return err
-	}
-	if ret.Error != "" {
-		return errors.New(ret.Error)
-	}
-	return nil
-}
-func vacuumVolume_Commit(urlLocation string, vid storage.VolumeId) error {
-	values := make(url.Values)
-	values.Add("volume", vid.String())
-	jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum_volume_commit", values)
-	if err != nil {
-		return err
-	}
-	var ret VacuumVolumeResult
-	if err := json.Unmarshal(jsonBlob, &ret); err != nil {
-		return err
-	}
-	if ret.Error != "" {
-		return errors.New(ret.Error)
-	}
-	return nil
-}

go/topology/volume_growth_test.go

@@ -1,135 +0,0 @@
-package topology
-
-import (
-	"encoding/json"
-	"fmt"
-	"testing"
-
-	"github.com/chrislusf/weed-fs/go/sequence"
-	"github.com/chrislusf/weed-fs/go/storage"
-)
-
-var topologyLayout = `
-{
-  "dc1":{
-    "rack1":{
-      "server111":{
-        "volumes":[
-          {"id":1, "size":12312},
-          {"id":2, "size":12312},
-          {"id":3, "size":12312}
-        ],
-        "limit":3
-      },
-      "server112":{
-        "volumes":[
-          {"id":4, "size":12312},
-          {"id":5, "size":12312},
-          {"id":6, "size":12312}
-        ],
-        "limit":10
-      }
-    },
-    "rack2":{
-      "server121":{
-        "volumes":[
-          {"id":4, "size":12312},
-          {"id":5, "size":12312},
-          {"id":6, "size":12312}
-        ],
-        "limit":4
-      },
-      "server122":{
-        "volumes":[],
-        "limit":4
-      },
-      "server123":{
-        "volumes":[
-          {"id":2, "size":12312},
-          {"id":3, "size":12312},
-          {"id":4, "size":12312}
-        ],
-        "limit":5
-      }
-    }
-  },
-  "dc2":{
-  },
-  "dc3":{
-    "rack2":{
-      "server321":{
-        "volumes":[
-          {"id":1, "size":12312},
-          {"id":3, "size":12312},
-          {"id":5, "size":12312}
-        ],
-        "limit":4
-      }
-    }
-  }
-}
-`
-
-func setup(topologyLayout string) *Topology {
-	var data interface{}
-	err := json.Unmarshal([]byte(topologyLayout), &data)
-	if err != nil {
-		fmt.Println("error:", err)
-	}
-	fmt.Println("data:", data)
-
-	//need to connect all nodes first before server adding volumes
-	topo, err := NewTopology("weedfs", "/etc/weedfs/weedfs.conf",
-		sequence.NewMemorySequencer(), 32*1024, 5)
-	if err != nil {
-		panic("error: " + err.Error())
-	}
-	mTopology := data.(map[string]interface{})
-	for dcKey, dcValue := range mTopology {
-		dc := NewDataCenter(dcKey)
-		dcMap := dcValue.(map[string]interface{})
-		topo.LinkChildNode(dc)
-		for rackKey, rackValue := range dcMap {
-			rack := NewRack(rackKey)
-			rackMap := rackValue.(map[string]interface{})
-			dc.LinkChildNode(rack)
-			for serverKey, serverValue := range rackMap {
-				server := NewDataNode(serverKey)
-				serverMap := serverValue.(map[string]interface{})
-				rack.LinkChildNode(server)
-				for _, v := range serverMap["volumes"].([]interface{}) {
-					m := v.(map[string]interface{})
-					vi := storage.VolumeInfo{
-						Id:      storage.VolumeId(int64(m["id"].(float64))),
-						Size:    uint64(m["size"].(float64)),
-						Version: storage.CurrentVersion}
-					server.AddOrUpdateVolume(vi)
-				}
-				server.UpAdjustMaxVolumeCountDelta(int(serverMap["limit"].(float64)))
-			}
-		}
-	}
-
-	return topo
-}
-
-func TestFindEmptySlotsForOneVolume(t *testing.T) {
-	topo := setup(topologyLayout)
-	vg := NewDefaultVolumeGrowth()
-	rp, _ := storage.NewReplicaPlacementFromString("002")
-	volumeGrowOption := &VolumeGrowOption{
-		Collection:       "",
-		ReplicaPlacement: rp,
-		DataCenter:       "dc1",
-		Rack:             "",
-		DataNode:         "",
-	}
-	servers, err := vg.findEmptySlotsForOneVolume(topo, volumeGrowOption)
-	if err != nil {
-		fmt.Println("finding empty slots error :", err)
-		t.Fail()
-	}
-	for _, server := range servers {
-		fmt.Println("assigned node :", server.Id())
-	}
-}

go/topology/volume_layout.go

@@ -1,214 +0,0 @@
-package topology
-
-import (
-	"errors"
-	"fmt"
-	"math/rand"
-	"sync"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/storage"
-)
-
-// mapping from volume to its locations, inverted from server to volume
-type VolumeLayout struct {
-	rp              *storage.ReplicaPlacement
-	ttl             *storage.TTL
-	vid2location    map[storage.VolumeId]*VolumeLocationList
-	writables       []storage.VolumeId // transient array of writable volume id
-	volumeSizeLimit uint64
-	accessLock      sync.Mutex
-}
-
-func NewVolumeLayout(rp *storage.ReplicaPlacement, ttl *storage.TTL, volumeSizeLimit uint64) *VolumeLayout {
-	return &VolumeLayout{
-		rp:              rp,
-		ttl:             ttl,
-		vid2location:    make(map[storage.VolumeId]*VolumeLocationList),
-		writables:       *new([]storage.VolumeId),
-		volumeSizeLimit: volumeSizeLimit,
-	}
-}
-
-func (vl *VolumeLayout) String() string {
-	return fmt.Sprintf("rp:%v, ttl:%v, vid2location:%v, writables:%v, volumeSizeLimit:%v", vl.rp, vl.ttl, vl.vid2location, vl.writables, vl.volumeSizeLimit)
-}
-
-func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
-	vl.accessLock.Lock()
-	defer vl.accessLock.Unlock()
-
-	if _, ok := vl.vid2location[v.Id]; !ok {
-		vl.vid2location[v.Id] = NewVolumeLocationList()
-	}
-	vl.vid2location[v.Id].Set(dn)
-	glog.V(4).Infoln("volume", v.Id, "added to dn", dn, "len", vl.vid2location[v.Id].Length(), "copy", v.ReplicaPlacement.GetCopyCount())
-	if vl.vid2location[v.Id].Length() == vl.rp.GetCopyCount() && vl.isWritable(v) {
-		vl.AddToWritable(v.Id)
-	} else {
-		vl.removeFromWritable(v.Id)
-	}
-}
-
-func (vl *VolumeLayout) UnRegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
-	vl.accessLock.Lock()
-	defer vl.accessLock.Unlock()
-
-	vl.removeFromWritable(v.Id)
-	delete(vl.vid2location, v.Id)
-}
-
-func (vl *VolumeLayout) AddToWritable(vid storage.VolumeId) {
-	for _, id := range vl.writables {
-		if vid == id {
-			return
-		}
-	}
-	vl.writables = append(vl.writables, vid)
-}
-
-func (vl *VolumeLayout) isWritable(v *storage.VolumeInfo) bool {
-	return uint64(v.Size) < vl.volumeSizeLimit &&
-		v.Version == storage.CurrentVersion &&
-		!v.ReadOnly
-}
-
-func (vl *VolumeLayout) Lookup(vid storage.VolumeId) []*DataNode {
-	if location := vl.vid2location[vid]; location != nil {
-		return location.list
-	}
-	return nil
-}
-
-func (vl *VolumeLayout) ListVolumeServers() (nodes []*DataNode) {
-	for _, location := range vl.vid2location {
-		nodes = append(nodes, location.list...)
-	}
-	return
-}
-
-func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (*storage.VolumeId, uint64, *VolumeLocationList, error) {
-	len_writers := len(vl.writables)
-	if len_writers <= 0 {
-		glog.V(0).Infoln("No more writable volumes!")
-		return nil, 0, nil, errors.New("No more writable volumes!")
-	}
-	if option.DataCenter == "" {
-		vid := vl.writables[rand.Intn(len_writers)]
-		locationList := vl.vid2location[vid]
-		if locationList != nil {
-			return &vid, count, locationList, nil
-		}
-		return nil, 0, nil, errors.New("Strangely vid " + vid.String() + " is on no machine!")
-	}
-	var vid storage.VolumeId
-	var locationList *VolumeLocationList
-	counter := 0
-	for _, v := range vl.writables {
-		volumeLocationList := vl.vid2location[v]
-		for _, dn := range volumeLocationList.list {
-			if dn.GetDataCenter().Id() == NodeId(option.DataCenter) {
-				if option.Rack != "" && dn.GetRack().Id() != NodeId(option.Rack) {
-					continue
-				}
-				if option.DataNode != "" && dn.Id() != NodeId(option.DataNode) {
-					continue
-				}
-				counter++
-				if rand.Intn(counter) < 1 {
-					vid, locationList = v, volumeLocationList
-				}
-			}
-		}
-	}
-	return &vid, count, locationList, nil
-}
-
-func (vl *VolumeLayout) GetActiveVolumeCount(option *VolumeGrowOption) int {
-	if option.DataCenter == "" {
-		return len(vl.writables)
-	}
-	counter := 0
-	for _, v := range vl.writables {
-		for _, dn := range vl.vid2location[v].list {
-			if dn.GetDataCenter().Id() == NodeId(option.DataCenter) {
-				if option.Rack != "" && dn.GetRack().Id() != NodeId(option.Rack) {
-					continue
-				}
-				if option.DataNode != "" && dn.Id() != NodeId(option.DataNode) {
-					continue
-				}
-				counter++
-			}
-		}
-	}
-	return counter
-}
-
-func (vl *VolumeLayout) removeFromWritable(vid storage.VolumeId) bool {
-	toDeleteIndex := -1
-	for k, id := range vl.writables {
-		if id == vid {
-			toDeleteIndex = k
-			break
-		}
-	}
-	if toDeleteIndex >= 0 {
-		glog.V(0).Infoln("Volume", vid, "becomes unwritable")
-		vl.writables = append(vl.writables[0:toDeleteIndex], vl.writables[toDeleteIndex+1:]...)
-		return true
-	}
-	return false
-}
-func (vl *VolumeLayout) setVolumeWritable(vid storage.VolumeId) bool {
-	for _, v := range vl.writables {
-		if v == vid {
-			return false
-		}
-	}
-	glog.V(0).Infoln("Volume", vid, "becomes writable")
-	vl.writables = append(vl.writables, vid)
-	return true
-}
-
-func (vl *VolumeLayout) SetVolumeUnavailable(dn *DataNode, vid storage.VolumeId) bool {
-	vl.accessLock.Lock()
-	defer vl.accessLock.Unlock()
-
-	if location, ok := vl.vid2location[vid]; ok {
-		if location.Remove(dn) {
-			if location.Length() < vl.rp.GetCopyCount() {
-				glog.V(0).Infoln("Volume", vid, "has", location.Length(), "replica, less than required", vl.rp.GetCopyCount())
-				return vl.removeFromWritable(vid)
-			}
-		}
-	}
-	return false
-}
-func (vl *VolumeLayout) SetVolumeAvailable(dn *DataNode, vid storage.VolumeId) bool {
-	vl.accessLock.Lock()
-	defer vl.accessLock.Unlock()
-
-	vl.vid2location[vid].Set(dn)
-	if vl.vid2location[vid].Length() >= vl.rp.GetCopyCount() {
-		return vl.setVolumeWritable(vid)
-	}
-	return false
-}
-
-func (vl *VolumeLayout) SetVolumeCapacityFull(vid storage.VolumeId) bool {
-	vl.accessLock.Lock()
-	defer vl.accessLock.Unlock()
-
-	// glog.V(0).Infoln("Volume", vid, "reaches full capacity.")
-	return vl.removeFromWritable(vid)
-}
-
-func (vl *VolumeLayout) ToMap() map[string]interface{} {
-	m := make(map[string]interface{})
-	m["replication"] = vl.rp.String()
-	m["ttl"] = vl.ttl.String()
-	m["writables"] = vl.writables
-	//m["locations"] = vl.vid2location
-	return m
-}

go/util/concurrent_read_map.go

@@ -1,38 +0,0 @@
-package util
-
-import (
-	"sync"
-)
-
-// A mostly for read map, which can thread-safely
-// initialize the map entries.
-type ConcurrentReadMap struct {
-	rmutex sync.RWMutex
-	mutex  sync.Mutex
-	Items  map[string]interface{}
-}
-
-func NewConcurrentReadMap() *ConcurrentReadMap {
-	return &ConcurrentReadMap{Items: make(map[string]interface{})}
-}
-
-func (m *ConcurrentReadMap) initMapEntry(key string, newEntry func() interface{}) (value interface{}) {
-	m.mutex.Lock()
-	defer m.mutex.Unlock()
-	if value, ok := m.Items[key]; ok {
-		return value
-	}
-	value = newEntry()
-	m.Items[key] = value
-	return value
-}
-
-func (m *ConcurrentReadMap) Get(key string, newEntry func() interface{}) interface{} {
-	m.rmutex.RLock()
-	if value, ok := m.Items[key]; ok {
-		m.rmutex.RUnlock()
-		return value
-	}
-	m.rmutex.RUnlock()
-	return m.initMapEntry(key, newEntry)
-}

go/util/config.go

@@ -1,130 +0,0 @@
-package util
-
-// Copyright 2011 Numerotron Inc.
-// Use of this source code is governed by an MIT-style license
-// that can be found in the LICENSE file.
-//
-// Developed at www.stathat.com by Patrick Crosby
-// Contact us on twitter with any questions:  twitter.com/stat_hat
-
-// The jconfig package provides a simple, basic configuration file parser using JSON.
-
-import (
-	"bytes"
-	"encoding/json"
-	"os"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-)
-
-type Config struct {
-	data     map[string]interface{}
-	filename string
-}
-
-func newConfig() *Config {
-	result := new(Config)
-	result.data = make(map[string]interface{})
-	return result
-}
-
-// Loads config information from a JSON file
-func LoadConfig(filename string) *Config {
-	result := newConfig()
-	result.filename = filename
-	err := result.parse()
-	if err != nil {
-		glog.Fatalf("error loading config file %s: %s", filename, err)
-	}
-	return result
-}
-
-// Loads config information from a JSON string
-func LoadConfigString(s string) *Config {
-	result := newConfig()
-	err := json.Unmarshal([]byte(s), &result.data)
-	if err != nil {
-		glog.Fatalf("error parsing config string %s: %s", s, err)
-	}
-	return result
-}
-
-func (c *Config) StringMerge(s string) {
-	next := LoadConfigString(s)
-	c.merge(next.data)
-}
-
-func (c *Config) LoadMerge(filename string) {
-	next := LoadConfig(filename)
-	c.merge(next.data)
-}
-
-func (c *Config) merge(ndata map[string]interface{}) {
-	for k, v := range ndata {
-		c.data[k] = v
-	}
-}
-
-func (c *Config) parse() error {
-	f, err := os.Open(c.filename)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	b := new(bytes.Buffer)
-	_, err = b.ReadFrom(f)
-	if err != nil {
-		return err
-	}
-	err = json.Unmarshal(b.Bytes(), &c.data)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Returns a string for the config variable key
-func (c *Config) GetString(key string) string {
-	result, present := c.data[key]
-	if !present {
-		return ""
-	}
-	return result.(string)
-}
-
-// Returns an int for the config variable key
-func (c *Config) GetInt(key string) int {
-	x, ok := c.data[key]
-	if !ok {
-		return -1
-	}
-	return int(x.(float64))
-}
-
-// Returns a float for the config variable key
-func (c *Config) GetFloat(key string) float64 {
-	x, ok := c.data[key]
-	if !ok {
-		return -1
-	}
-	return x.(float64)
-}
-
-// Returns a bool for the config variable key
-func (c *Config) GetBool(key string) bool {
-	x, ok := c.data[key]
-	if !ok {
-		return false
-	}
-	return x.(bool)
-}
-
-// Returns an array for the config variable key
-func (c *Config) GetArray(key string) []interface{} {
-	result, present := c.data[key]
-	if !present {
-		return []interface{}(nil)
-	}
-	return result.([]interface{})
-}

go/util/constants.go

@@ -1,5 +0,0 @@
-package util
-
-const (
-	VERSION = "0.69 20150413"
-)

go/util/file_util.go

@@ -1,38 +0,0 @@
-package util
-
-import (
-	"bufio"
-	"errors"
-	"os"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-)
-
-func TestFolderWritable(folder string) (err error) {
-	fileInfo, err := os.Stat(folder)
-	if err != nil {
-		return err
-	}
-	if !fileInfo.IsDir() {
-		return errors.New("Not a valid folder!")
-	}
-	perm := fileInfo.Mode().Perm()
-	glog.V(0).Infoln("Folder", folder, "Permission:", perm)
-	if 0200&perm != 0 {
-		return nil
-	}
-	return errors.New("Not writable!")
-}
-
-func Readln(r *bufio.Reader) ([]byte, error) {
-	var (
-		isPrefix = true
-		err      error
-		line, ln []byte
-	)
-	for isPrefix && err == nil {
-		line, isPrefix, err = r.ReadLine()
-		ln = append(ln, line...)
-	}
-	return ln, err
-}

go/util/http_util.go

@@ -1,106 +0,0 @@
-package util
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"strings"
-
-	"github.com/chrislusf/weed-fs/go/security"
-)
-
-var (
-	client    *http.Client
-	Transport *http.Transport
-)
-
-func init() {
-	Transport = &http.Transport{
-		MaxIdleConnsPerHost: 1024,
-	}
-	client = &http.Client{Transport: Transport}
-}
-
-func PostBytes(url string, body []byte) ([]byte, error) {
-	r, err := client.Post(url, "application/octet-stream", bytes.NewReader(body))
-	if err != nil {
-		return nil, fmt.Errorf("Post to %s: %v", url, err)
-	}
-	defer r.Body.Close()
-	b, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		return nil, fmt.Errorf("Read response body: %v", err)
-	}
-	return b, nil
-}
-
-func Post(url string, values url.Values) ([]byte, error) {
-	r, err := client.PostForm(url, values)
-	if err != nil {
-		return nil, err
-	}
-	defer r.Body.Close()
-	b, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		return nil, err
-	}
-	return b, nil
-}
-
-func Get(url string) ([]byte, error) {
-	r, err := client.Get(url)
-	if err != nil {
-		return nil, err
-	}
-	defer r.Body.Close()
-	b, err := ioutil.ReadAll(r.Body)
-	if r.StatusCode != 200 {
-		return nil, fmt.Errorf("%s: %s", url, r.Status)
-	}
-	if err != nil {
-		return nil, err
-	}
-	return b, nil
-}
-
-func Delete(url string, jwt security.EncodedJwt) error {
-	req, err := http.NewRequest("DELETE", url, nil)
-	if jwt != "" {
-		req.Header.Set("Authorization", "BEARER "+string(jwt))
-	}
-	if err != nil {
-		return err
-	}
-	resp, e := client.Do(req)
-	if e != nil {
-		return e
-	}
-	defer resp.Body.Close()
-	if _, err := ioutil.ReadAll(resp.Body); err != nil {
-		return err
-	}
-	return nil
-}
-
-func DownloadUrl(fileUrl string) (filename string, content []byte, e error) {
-	response, err := client.Get(fileUrl)
-	if err != nil {
-		return "", nil, err
-	}
-	defer response.Body.Close()
-	contentDisposition := response.Header["Content-Disposition"]
-	if len(contentDisposition) > 0 {
-		if strings.HasPrefix(contentDisposition[0], "filename=") {
-			filename = contentDisposition[0][len("filename="):]
-			filename = strings.Trim(filename, "\"")
-		}
-	}
-	content, e = ioutil.ReadAll(response.Body)
-	return
-}
-
-func Do(req *http.Request) (resp *http.Response, err error) {
-	return client.Do(req)
-}

go/util/parse.go

@@ -1,16 +0,0 @@
-package util
-
-import (
-	"strconv"
-)
-
-func ParseInt(text string, defaultValue int) int {
-	count, parseError := strconv.ParseUint(text, 10, 64)
-	if parseError != nil {
-		if len(text) > 0 {
-			return 0
-		}
-		return defaultValue
-	}
-	return int(count)
-}

go/weed/compact.go

@@ -1,46 +0,0 @@
-package main
-
-import (
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/storage"
-)
-
-func init() {
-	cmdCompact.Run = runCompact // break init cycle
-	cmdCompact.IsDebug = cmdCompact.Flag.Bool("debug", false, "enable debug mode")
-}
-
-var cmdCompact = &Command{
-	UsageLine: "compact -dir=/tmp -volumeId=234",
-	Short:     "run weed tool compact on volume file",
-	Long: `Force an compaction to remove deleted files from volume files.
-  The compacted .dat file is stored as .cpd file.
-  The compacted .idx file is stored as .cpx file.
-
-  `,
-}
-
-var (
-	compactVolumePath       = cmdCompact.Flag.String("dir", "/tmp", "data directory to store files")
-	compactVolumeCollection = cmdCompact.Flag.String("collection", "", "volume collection name")
-	compactVolumeId         = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.")
-)
-
-func runCompact(cmd *Command, args []string) bool {
-
-	if *compactVolumeId == -1 {
-		return false
-	}
-
-	vid := storage.VolumeId(*compactVolumeId)
-	v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid,
-		storage.NeedleMapInMemory, nil, nil)
-	if err != nil {
-		glog.Fatalf("Load Volume [ERROR] %s\n", err)
-	}
-	if err = v.Compact(); err != nil {
-		glog.Fatalf("Compact Volume [ERROR] %s\n", err)
-	}
-
-	return true
-}

go/weed/cover.out

@@ -1,477 +0,0 @@
-mode: set
-github.com/chrislusf/weed-fs/go/weed/weed.go:41.27,43.20 2 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:46.2,46.17 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:43.20,45.3 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:49.13,56.19 6 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:60.2,60.23 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:71.2,71.31 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:88.2,90.8 3 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:56.19,58.3 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:60.23,62.32 2 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:68.3,68.9 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:62.32,63.65 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:63.65,66.5 2 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:71.31,72.46 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:72.46,73.28 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:74.4,77.27 4 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:83.4,84.10 2 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:73.28,73.43 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:77.27,82.5 4 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:114.55,118.43 4 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:118.43,120.3 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:123.34,124.13 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:127.2,128.43 2 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:124.13,126.3 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:131.30,133.2 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:135.14,140.2 4 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:143.26,144.20 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:149.2,149.20 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:154.2,156.31 2 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:164.2,165.12 2 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:144.20,148.3 2 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:149.20,152.3 2 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:156.31,157.24 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:157.24,161.4 2 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:170.23,172.2 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:174.13,175.32 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:178.2,178.21 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:175.32,177.3 1 0
-github.com/chrislusf/weed-fs/go/weed/weed.go:181.35,183.2 1 0
-github.com/chrislusf/weed-fs/go/weed/master.go:17.13,19.2 1 1
-github.com/chrislusf/weed-fs/go/weed/master.go:50.50,51.18 1 0
-github.com/chrislusf/weed-fs/go/weed/master.go:54.2,55.61 2 0
-github.com/chrislusf/weed-fs/go/weed/master.go:58.2,58.34 1 0
-github.com/chrislusf/weed-fs/go/weed/master.go:62.2,73.14 6 0
-github.com/chrislusf/weed-fs/go/weed/master.go:77.2,77.12 1 0
-github.com/chrislusf/weed-fs/go/weed/master.go:91.2,91.44 1 0
-github.com/chrislusf/weed-fs/go/weed/master.go:94.2,94.13 1 0
-github.com/chrislusf/weed-fs/go/weed/master.go:51.18,53.3 1 0
-github.com/chrislusf/weed-fs/go/weed/master.go:55.61,57.3 1 0
-github.com/chrislusf/weed-fs/go/weed/master.go:58.34,60.3 1 0
-github.com/chrislusf/weed-fs/go/weed/master.go:73.14,75.3 1 0
-github.com/chrislusf/weed-fs/go/weed/master.go:77.12,80.24 3 0
-github.com/chrislusf/weed-fs/go/weed/master.go:83.3,84.25 2 0
-github.com/chrislusf/weed-fs/go/weed/master.go:87.3,88.31 2 0
-github.com/chrislusf/weed-fs/go/weed/master.go:80.24,82.4 1 0
-github.com/chrislusf/weed-fs/go/weed/master.go:84.25,86.4 1 0
-github.com/chrislusf/weed-fs/go/weed/master.go:91.44,93.3 1 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:11.13,13.2 1 1
-github.com/chrislusf/weed-fs/go/weed/shell.go:25.53,29.19 4 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:38.2,38.28 1 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:46.2,46.34 1 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:55.2,56.6 2 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:29.19,31.47 2 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:34.3,34.34 1 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:31.47,33.4 1 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:34.34,36.4 1 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:38.28,40.17 2 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:44.3,44.13 1 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:40.17,43.4 2 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:46.34,47.16 1 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:52.3,52.11 1 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:47.16,48.48 1 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:48.48,50.5 1 0
-github.com/chrislusf/weed-fs/go/weed/shell.go:56.6,60.3 3 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:23.13,34.2 10 1
-github.com/chrislusf/weed-fs/go/weed/upload.go:59.50,61.37 2 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:102.2,102.13 1 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:61.37,62.23 1 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:65.3,65.82 1 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:62.23,64.4 1 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:65.82,66.18 1 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:89.4,89.14 1 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:66.18,67.22 1 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:67.22,68.24 1 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:73.6,74.18 2 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:77.6,82.18 4 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:68.24,69.70 1 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:69.70,71.8 1 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:74.18,76.7 1 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:82.18,84.7 1 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:86.5,88.5 1 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:91.3,93.15 2 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:96.3,100.29 3 0
-github.com/chrislusf/weed-fs/go/weed/upload.go:93.15,95.4 1 0
-github.com/chrislusf/weed-fs/go/weed/signal_handling.go:11.29,22.12 3 0
-github.com/chrislusf/weed-fs/go/weed/signal_handling.go:22.12,23.28 1 0
-github.com/chrislusf/weed-fs/go/weed/signal_handling.go:23.28,26.4 2 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:38.13,52.2 13 1
-github.com/chrislusf/weed-fs/go/weed/volume.go:68.50,69.19 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:72.2,77.44 4 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:84.2,84.46 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:87.2,87.35 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:94.2,94.34 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:98.2,98.17 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:101.2,101.24 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:105.2,105.24 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:108.2,112.27 4 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:116.2,127.14 5 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:130.2,130.27 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:144.2,144.21 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:148.2,148.52 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:151.2,151.13 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:69.19,71.3 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:77.44,78.50 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:78.50,80.4 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:80.4,82.4 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:84.46,86.3 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:87.35,88.57 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:88.57,90.4 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:94.34,96.3 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:98.17,100.3 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:101.24,103.3 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:105.24,107.3 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:112.27,114.3 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:127.14,129.3 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:130.27,134.15 4 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:137.3,137.13 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:134.15,136.4 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:137.13,138.66 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:138.66,140.5 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:144.21,146.3 1 0
-github.com/chrislusf/weed-fs/go/weed/volume.go:148.52,150.3 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:20.13,25.2 4 1
-github.com/chrislusf/weed-fs/go/weed/download.go:41.52,42.27 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:78.2,78.13 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:42.27,44.15 2 0
-github.com/chrislusf/weed-fs/go/weed/download.go:48.3,48.21 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:51.3,51.43 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:44.15,46.12 2 0
-github.com/chrislusf/weed-fs/go/weed/download.go:48.21,50.4 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:51.43,55.18 4 0
-github.com/chrislusf/weed-fs/go/weed/download.go:59.4,60.32 2 0
-github.com/chrislusf/weed-fs/go/weed/download.go:55.18,57.13 2 0
-github.com/chrislusf/weed-fs/go/weed/download.go:60.32,63.19 3 0
-github.com/chrislusf/weed-fs/go/weed/download.go:66.5,66.36 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:69.5,69.19 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:63.19,65.6 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:66.36,68.6 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:69.19,71.11 2 0
-github.com/chrislusf/weed-fs/go/weed/download.go:74.4,76.4 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:81.91,83.24 2 0
-github.com/chrislusf/weed-fs/go/weed/download.go:86.2,87.8 2 0
-github.com/chrislusf/weed-fs/go/weed/download.go:83.24,85.3 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:90.70,92.16 2 0
-github.com/chrislusf/weed-fs/go/weed/download.go:95.2,97.33 3 0
-github.com/chrislusf/weed-fs/go/weed/download.go:100.2,100.12 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:92.16,94.3 1 0
-github.com/chrislusf/weed-fs/go/weed/download.go:97.33,99.3 1 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:12.13,14.2 1 1
-github.com/chrislusf/weed-fs/go/weed/fix.go:30.47,32.24 1 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:36.2,37.32 2 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:40.2,41.16 2 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:44.2,50.116 5 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:63.2,63.16 1 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:67.2,67.13 1 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:32.24,34.3 1 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:37.32,39.3 1 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:41.16,43.3 1 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:50.116,52.3 1 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:52.56,54.17 2 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:61.3,61.13 1 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:54.17,57.4 2 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:57.4,60.4 2 0
-github.com/chrislusf/weed-fs/go/weed/fix.go:63.16,65.3 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:28.13,30.2 1 1
-github.com/chrislusf/weed-fs/go/weed/server.go:77.13,89.2 11 1
-github.com/chrislusf/weed-fs/go/weed/server.go:91.50,93.37 2 0
-github.com/chrislusf/weed-fs/go/weed/server.go:102.2,102.34 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:106.2,108.49 2 0
-github.com/chrislusf/weed-fs/go/weed/server.go:112.2,112.28 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:116.2,116.23 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:119.2,124.44 5 0
-github.com/chrislusf/weed-fs/go/weed/server.go:131.2,131.36 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:134.2,134.33 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:140.2,140.29 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:143.2,143.22 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:152.2,152.67 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:156.2,156.34 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:160.2,160.22 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:186.2,192.12 5 0
-github.com/chrislusf/weed-fs/go/weed/server.go:224.2,226.28 3 0
-github.com/chrislusf/weed-fs/go/weed/server.go:229.2,232.27 4 0
-github.com/chrislusf/weed-fs/go/weed/server.go:235.2,247.20 4 0
-github.com/chrislusf/weed-fs/go/weed/server.go:250.2,250.27 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:264.2,264.21 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:269.2,269.58 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:273.2,273.13 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:93.37,95.17 2 0
-github.com/chrislusf/weed-fs/go/weed/server.go:98.3,99.31 2 0
-github.com/chrislusf/weed-fs/go/weed/server.go:95.17,97.4 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:102.34,104.3 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:108.49,110.3 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:112.28,114.3 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:116.23,118.3 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:124.44,125.50 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:125.50,127.4 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:127.4,129.4 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:131.36,133.3 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:134.33,135.57 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:135.57,137.4 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:140.29,142.3 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:143.22,144.30 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:148.3,148.68 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:144.30,147.4 2 0
-github.com/chrislusf/weed-fs/go/weed/server.go:148.68,150.4 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:152.67,154.3 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:156.34,158.3 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:160.22,161.13 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:161.13,169.22 3 0
-github.com/chrislusf/weed-fs/go/weed/server.go:172.4,177.16 3 0
-github.com/chrislusf/weed-fs/go/weed/server.go:180.4,180.51 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:169.22,171.5 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:177.16,179.5 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:180.51,182.5 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:192.12,201.15 5 0
-github.com/chrislusf/weed-fs/go/weed/server.go:205.3,205.13 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:218.3,219.51 2 0
-github.com/chrislusf/weed-fs/go/weed/server.go:201.15,203.4 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:205.13,210.26 5 0
-github.com/chrislusf/weed-fs/go/weed/server.go:213.4,215.21 3 0
-github.com/chrislusf/weed-fs/go/weed/server.go:210.26,212.5 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:219.51,221.4 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:226.28,228.3 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:232.27,234.3 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:247.20,249.3 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:250.27,254.15 4 0
-github.com/chrislusf/weed-fs/go/weed/server.go:257.3,257.13 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:254.15,256.4 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:257.13,258.66 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:258.66,260.5 1 0
-github.com/chrislusf/weed-fs/go/weed/server.go:264.21,267.3 2 0
-github.com/chrislusf/weed-fs/go/weed/server.go:269.58,271.3 1 0
-github.com/chrislusf/weed-fs/go/weed/filer.go:32.13,46.2 12 1
-github.com/chrislusf/weed-fs/go/weed/filer.go:70.49,72.56 1 0
-github.com/chrislusf/weed-fs/go/weed/filer.go:76.2,83.20 3 0
-github.com/chrislusf/weed-fs/go/weed/filer.go:86.2,91.14 3 0
-github.com/chrislusf/weed-fs/go/weed/filer.go:94.2,94.49 1 0
-github.com/chrislusf/weed-fs/go/weed/filer.go:98.2,98.13 1 0
-github.com/chrislusf/weed-fs/go/weed/filer.go:72.56,74.3 1 0
-github.com/chrislusf/weed-fs/go/weed/filer.go:83.20,85.3 1 0
-github.com/chrislusf/weed-fs/go/weed/filer.go:91.14,93.3 1 0
-github.com/chrislusf/weed-fs/go/weed/filer.go:94.49,96.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:46.13,64.2 17 1
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:104.53,106.19 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:109.2,110.25 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:119.2,119.14 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:123.2,123.13 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:127.2,127.13 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:106.19,108.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:110.25,112.17 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:115.3,116.31 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:112.17,114.4 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:119.14,121.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:123.13,125.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:130.20,136.38 6 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:140.2,143.40 4 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:146.2,154.25 9 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:136.38,139.3 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:143.40,145.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:157.19,165.38 8 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:169.2,175.24 7 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:165.38,168.3 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:183.71,189.25 5 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:207.2,207.25 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:235.2,236.25 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:189.25,191.13 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:191.13,193.38 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:193.38,194.39 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:197.5,198.52 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:194.39,196.6 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:198.52,200.6 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:200.6,202.6 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:207.25,211.93 4 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:211.93,213.61 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:226.4,227.29 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:213.61,214.45 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:220.5,221.30 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:214.45,217.6 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:217.6,219.6 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:222.5,225.5 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:227.29,229.5 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:230.4,233.4 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:239.53,242.34 3 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:242.34,243.20 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:246.3,246.20 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:249.3,249.28 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:252.3,255.43 4 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:267.3,267.42 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:243.20,244.12 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:246.20,247.12 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:249.28,251.4 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:255.43,257.47 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:265.4,265.21 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:257.47,258.65 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:258.65,259.32 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:259.32,262.7 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:267.42,269.51 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:269.51,273.5 3 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:273.5,276.5 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:277.4,280.4 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:284.86,286.16 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:289.2,291.6 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:286.16,288.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:291.6,292.10 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:293.3,295.10 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:296.3,298.28 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:303.63,305.16 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:308.2,311.23 3 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:335.2,335.23 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:305.16,307.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:311.23,312.7 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:312.7,313.42 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:313.42,315.5 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:315.5,316.10 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:319.3,321.7 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:328.3,328.21 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:321.7,322.42 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:322.42,324.5 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:324.5,325.10 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:328.21,329.41 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:329.41,331.5 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:361.29,367.2 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:369.44,371.15 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:371.15,373.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:373.3,373.32 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:373.32,375.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:375.3,377.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:380.70,384.6 4 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:384.6,385.10 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:386.3,388.10 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:389.3,391.43 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:396.4,401.72 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:391.43,395.5 3 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:406.30,408.41 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:414.2,424.35 11 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:436.2,437.39 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:446.2,448.35 3 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:454.2,454.39 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:458.2,465.40 7 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:468.2,471.35 4 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:481.2,482.39 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:408.41,413.3 4 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:424.35,427.20 3 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:427.20,428.15 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:431.4,431.15 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:428.15,430.5 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:431.15,433.5 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:437.39,439.26 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:442.3,442.26 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:439.26,441.4 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:442.26,444.4 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:448.35,449.20 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:449.20,452.4 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:454.39,457.3 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:465.40,467.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:471.35,473.104 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:473.104,476.89 3 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:476.89,478.5 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:482.39,484.87 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:484.87,487.89 3 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:487.89,489.5 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:500.56,501.17 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:504.2,504.28 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:509.2,509.12 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:514.2,515.8 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:501.17,503.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:504.28,506.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:506.3,508.3 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:509.12,510.26 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:510.26,512.4 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:518.64,521.15 3 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:532.2,532.20 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:521.15,523.24 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:526.3,527.15 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:530.3,530.16 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:523.24,525.4 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:527.15,529.4 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:535.46,541.29 2 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:545.2,545.16 1 0
-github.com/chrislusf/weed-fs/go/weed/benchmark.go:541.29,544.3 2 0
-github.com/chrislusf/weed-fs/go/weed/command.go:32.33,35.12 3 0
-github.com/chrislusf/weed-fs/go/weed/command.go:38.2,38.13 1 0
-github.com/chrislusf/weed-fs/go/weed/command.go:35.12,37.3 1 0
-github.com/chrislusf/weed-fs/go/weed/command.go:41.27,48.2 6 0
-github.com/chrislusf/weed-fs/go/weed/command.go:52.35,54.2 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:18.13,20.2 1 1
-github.com/chrislusf/weed-fs/go/weed/export.go:48.50,50.27 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:54.2,55.17 2 0
-github.com/chrislusf/weed-fs/go/weed/export.go:84.2,85.29 2 0
-github.com/chrislusf/weed-fs/go/weed/export.go:88.2,90.16 3 0
-github.com/chrislusf/weed-fs/go/weed/export.go:93.2,96.16 3 0
-github.com/chrislusf/weed-fs/go/weed/export.go:100.2,102.116 2 0
-github.com/chrislusf/weed-fs/go/weed/export.go:120.2,120.16 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:123.2,123.13 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:50.27,52.3 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:55.17,56.56 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:61.3,61.68 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:66.3,67.19 2 0
-github.com/chrislusf/weed-fs/go/weed/export.go:74.3,81.33 5 0
-github.com/chrislusf/weed-fs/go/weed/export.go:56.56,59.4 2 0
-github.com/chrislusf/weed-fs/go/weed/export.go:61.68,64.4 2 0
-github.com/chrislusf/weed-fs/go/weed/export.go:67.19,69.4 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:69.4,70.46 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:70.46,72.5 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:85.29,87.3 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:90.16,92.3 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:96.16,98.3 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:102.116,105.3 2 0
-github.com/chrislusf/weed-fs/go/weed/export.go:105.55,109.56 3 0
-github.com/chrislusf/weed-fs/go/weed/export.go:118.3,118.13 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:109.56,111.4 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:111.4,112.11 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:112.11,114.5 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:114.5,116.5 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:120.16,122.3 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:133.91,135.18 2 0
-github.com/chrislusf/weed-fs/go/weed/export.go:170.2,170.8 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:135.18,143.17 2 0
-github.com/chrislusf/weed-fs/go/weed/export.go:146.3,148.45 2 0
-github.com/chrislusf/weed-fs/go/weed/export.go:152.3,153.54 2 0
-github.com/chrislusf/weed-fs/go/weed/export.go:156.3,156.31 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:143.17,145.4 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:148.45,150.4 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:153.54,155.4 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:157.3,159.34 2 0
-github.com/chrislusf/weed-fs/go/weed/export.go:162.3,168.4 1 0
-github.com/chrislusf/weed-fs/go/weed/export.go:159.34,161.4 1 0
-github.com/chrislusf/weed-fs/go/weed/version.go:17.51,18.20 1 0
-github.com/chrislusf/weed-fs/go/weed/version.go:22.2,23.13 2 0
-github.com/chrislusf/weed-fs/go/weed/version.go:18.20,20.3 1 0
-github.com/chrislusf/weed-fs/go/weed/compact.go:8.13,11.2 2 1
-github.com/chrislusf/weed-fs/go/weed/compact.go:29.51,31.28 1 0
-github.com/chrislusf/weed-fs/go/weed/compact.go:35.2,37.16 3 0
-github.com/chrislusf/weed-fs/go/weed/compact.go:40.2,40.35 1 0
-github.com/chrislusf/weed-fs/go/weed/compact.go:44.2,44.13 1 0
-github.com/chrislusf/weed-fs/go/weed/compact.go:31.28,33.3 1 0
-github.com/chrislusf/weed-fs/go/weed/compact.go:37.16,39.3 1 0
-github.com/chrislusf/weed-fs/go/weed/compact.go:40.35,42.3 1 0
-github.com/chrislusf/weed-fs/go/weed/mount.go:12.13,17.2 4 1
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:19.49,21.29 2 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:26.2,27.16 2 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:32.2,32.21 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:37.2,38.16 2 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:43.2,44.37 2 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:48.2,48.13 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:21.29,24.3 2 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:27.16,30.3 2 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:32.21,35.3 2 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:38.16,40.3 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:44.37,46.3 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:56.30,58.2 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:59.58,61.2 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:68.33,70.2 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:72.74,74.14 2 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:77.2,77.33 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:80.2,80.55 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:74.14,76.3 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:77.33,79.3 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:85.36,87.2 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:89.69,91.79 2 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:97.2,97.78 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:106.2,106.17 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:91.79,92.38 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:92.38,95.4 2 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:97.78,98.33 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:98.33,99.64 1 0
-github.com/chrislusf/weed-fs/go/weed/mount_std.go:99.64,102.5 2 0

go/weed/download.go

@@ -1,101 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path"
-	"strings"
-
-	"github.com/chrislusf/weed-fs/go/operation"
-	"github.com/chrislusf/weed-fs/go/util"
-)
-
-var (
-	downloadReplication *string
-	downloadDir         *string
-)
-
-func init() {
-	cmdDownload.Run = runDownload // break init cycle
-	cmdDownload.IsDebug = cmdDownload.Flag.Bool("debug", false, "verbose debug information")
-	server = cmdDownload.Flag.String("server", "localhost:9333", "weedfs master location")
-	downloadDir = cmdDownload.Flag.String("dir", ".", "Download the whole folder recursively if specified.")
-}
-
-var cmdDownload = &Command{
-	UsageLine: "download -server=localhost:9333 -dir=one_directory fid1 [fid2 fid3 ...]",
-	Short:     "download files by file id",
-	Long: `download files by file id.
-  
-  Usually you just need to use curl to lookup the file's volume server, and then download them directly.
-  This download tool combine the two steps into one.
-
-  What's more, if you use "weed upload -maxMB=..." option to upload a big file divided into chunks, you can
-  use this tool to download the chunks and merge them automatically.
-
-  `,
-}
-
-func runDownload(cmd *Command, args []string) bool {
-	for _, fid := range args {
-		filename, content, e := fetchFileId(*server, fid)
-		if e != nil {
-			fmt.Println("Fetch Error:", e)
-			continue
-		}
-		if filename == "" {
-			filename = fid
-		}
-		if strings.HasSuffix(filename, "-list") {
-			filename = filename[0 : len(filename)-len("-list")]
-			fids := strings.Split(string(content), "\n")
-			f, err := os.OpenFile(path.Join(*downloadDir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
-			if err != nil {
-				fmt.Println("File Creation Error:", e)
-				continue
-			}
-			defer f.Close()
-			for _, partId := range fids {
-				var n int
-				_, part, err := fetchFileId(*server, partId)
-				if err == nil {
-					n, err = f.Write(part)
-				}
-				if err == nil && n < len(part) {
-					err = io.ErrShortWrite
-				}
-				if err != nil {
-					fmt.Println("File Write Error:", err)
-					break
-				}
-			}
-		} else {
-			ioutil.WriteFile(path.Join(*downloadDir, filename), content, os.ModePerm)
-		}
-	}
-	return true
-}
-
-func fetchFileId(server string, fileId string) (filename string, content []byte, e error) {
-	fileUrl, lookupError := operation.LookupFileId(server, fileId)
-	if lookupError != nil {
-		return "", nil, lookupError
-	}
-	filename, content, e = util.DownloadUrl(fileUrl)
-	return
-}
-
-func WriteFile(filename string, data []byte, perm os.FileMode) error {
-	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
-	if err != nil {
-		return err
-	}
-	n, err := f.Write(data)
-	f.Close()
-	if err == nil && n < len(data) {
-		err = io.ErrShortWrite
-	}
-	return err
-}

go/weed/export.go

@@ -1,172 +0,0 @@
-package main
-
-import (
-	"archive/tar"
-	"bytes"
-	"fmt"
-	"os"
-	"path"
-	"strconv"
-	"strings"
-	"text/template"
-	"time"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/storage"
-)
-
-func init() {
-	cmdExport.Run = runExport // break init cycle
-}
-
-const (
-	defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}`
-)
-
-var cmdExport = &Command{
-	UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}}",
-	Short:     "list or export files from one volume data file",
-	Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified.
-
-	The format of file name in the tar file can be customized. Default is {{.Mime}}/{{.Id}}:{{.Name}}. Also available is {{.Key}}.
-
-  `,
-}
-
-var (
-	exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "input data directory to store volume data files")
-	exportCollection = cmdExport.Flag.String("collection", "", "the volume collection name")
-	exportVolumeId   = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
-	dest             = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
-	format           = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}")
-	tarFh            *tar.Writer
-	tarHeader        tar.Header
-	fnTmpl           *template.Template
-	fnTmplBuf        = bytes.NewBuffer(nil)
-)
-
-func runExport(cmd *Command, args []string) bool {
-
-	if *exportVolumeId == -1 {
-		return false
-	}
-
-	var err error
-	if *dest != "" {
-		if *dest != "-" && !strings.HasSuffix(*dest, ".tar") {
-			fmt.Println("the output file", *dest, "should be '-' or end with .tar")
-			return false
-		}
-
-		if fnTmpl, err = template.New("name").Parse(*format); err != nil {
-			fmt.Println("cannot parse format " + *format + ": " + err.Error())
-			return false
-		}
-
-		var fh *os.File
-		if *dest == "-" {
-			fh = os.Stdout
-		} else {
-			if fh, err = os.Create(*dest); err != nil {
-				glog.Fatalf("cannot open output tar %s: %s", *dest, err)
-			}
-		}
-		defer fh.Close()
-		tarFh = tar.NewWriter(fh)
-		defer tarFh.Close()
-		t := time.Now()
-		tarHeader = tar.Header{Mode: 0644,
-			ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
-			Typeflag:   tar.TypeReg,
-			AccessTime: t, ChangeTime: t}
-	}
-
-	fileName := strconv.Itoa(*exportVolumeId)
-	if *exportCollection != "" {
-		fileName = *exportCollection + "_" + fileName
-	}
-	vid := storage.VolumeId(*exportVolumeId)
-	indexFile, err := os.OpenFile(path.Join(*exportVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
-	if err != nil {
-		glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
-	}
-	defer indexFile.Close()
-
-	nm, err := storage.LoadNeedleMap(indexFile)
-	if err != nil {
-		glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
-	}
-
-	var version storage.Version
-
-	err = storage.ScanVolumeFile(*exportVolumePath, *exportCollection, vid,
-		storage.NeedleMapInMemory,
-		func(superBlock storage.SuperBlock) error {
-			version = superBlock.Version()
-			return nil
-		}, true, func(n *storage.Needle, offset int64) error {
-			nv, ok := nm.Get(n.Id)
-			glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
-				n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped(), ok, nv)
-			if ok && nv.Size > 0 && int64(nv.Offset)*8 == offset {
-				return walker(vid, n, version)
-			}
-			if !ok {
-				glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
-			} else {
-				glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
-			}
-			return nil
-		})
-	if err != nil {
-		glog.Fatalf("Export Volume File [ERROR] %s\n", err)
-	}
-	return true
-}
-
-type nameParams struct {
-	Name string
-	Id   uint64
-	Mime string
-	Key  string
-}
-
-func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) {
-	key := storage.NewFileIdFromNeedle(vid, n).String()
-	if tarFh != nil {
-		fnTmplBuf.Reset()
-		if err = fnTmpl.Execute(fnTmplBuf,
-			nameParams{Name: string(n.Name),
-				Id:   n.Id,
-				Mime: string(n.Mime),
-				Key:  key,
-			},
-		); err != nil {
-			return err
-		}
-		nm := fnTmplBuf.String()
-
-		if n.IsGzipped() && path.Ext(nm) != ".gz" {
-			nm = nm + ".gz"
-		}
-
-		tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
-		if err = tarFh.WriteHeader(&tarHeader); err != nil {
-			return err
-		}
-		_, err = tarFh.Write(n.Data)
-	} else {
-		size := n.DataSize
-		if version == storage.Version1 {
-			size = n.Size
-		}
-		fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n",
-			key,
-			n.Name,
-			size,
-			n.IsGzipped(),
-			n.Mime,
-		)
-	}
-	return
-}

go/weed/filer.go

@@ -1,101 +0,0 @@
-package main
-
-import (
-	"net/http"
-	"os"
-	"strconv"
-	"time"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
-	"github.com/chrislusf/weed-fs/go/weed/weed_server"
-)
-
-var (
-	f FilerOptions
-)
-
-type FilerOptions struct {
-	master                  *string
-	port                    *int
-	collection              *string
-	defaultReplicaPlacement *string
-	dir                     *string
-	redirectOnRead          *bool
-	disableDirListing       *bool
-	secretKey               *string
-	cassandra_server        *string
-	cassandra_keyspace      *string
-	redis_server            *string
-	redis_database          *int
-}
-
-func init() {
-	cmdFiler.Run = runFiler // break init cycle
-	f.master = cmdFiler.Flag.String("master", "localhost:9333", "master server location")
-	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection")
-	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
-	f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data")
-	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
-	f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
-	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
-	f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server")
-	f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server")
-	f.redis_server = cmdFiler.Flag.String("redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379")
-	f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server")
-	f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
-
-}
-
-var cmdFiler = &Command{
-	UsageLine: "filer -port=8888 -dir=/tmp -master=<ip:port>",
-	Short:     "start a file server that points to a master server",
-	Long: `start a file server which accepts REST operation for any files.
-
-	//create or overwrite the file, the directories /path/to will be automatically created
-	POST /path/to/file
-	//get the file content
-	GET /path/to/file
-	//create or overwrite the file, the filename in the multipart request will be used
-	POST /path/to/
-	//return a json format subdirectory and files listing
-	GET /path/to/
-
-  Current <fullpath~fileid> mapping metadata store is local embedded leveldb.
-  It should be highly scalable to hundreds of millions of files on a modest machine.
-
-  Future we will ensure it can avoid of being SPOF.
-
-  `,
-}
-
-func runFiler(cmd *Command, args []string) bool {
-
-	if err := util.TestFolderWritable(*f.dir); err != nil {
-		glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err)
-	}
-
-	r := http.NewServeMux()
-	_, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection,
-		*f.defaultReplicaPlacement, *f.redirectOnRead, *f.disableDirListing,
-		*f.secretKey,
-		*f.cassandra_server, *f.cassandra_keyspace,
-		*f.redis_server, *f.redis_database,
-	)
-	if nfs_err != nil {
-		glog.Fatalf("Filer startup error: %v", nfs_err)
-	}
-	glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*f.port))
-	filerListener, e := util.NewListener(
-		":"+strconv.Itoa(*f.port),
-		time.Duration(10)*time.Second,
-	)
-	if e != nil {
-		glog.Fatalf("Filer listener error: %v", e)
-	}
-	if e := http.Serve(filerListener, r); e != nil {
-		glog.Fatalf("Filer Fail to serve: %v", e)
-	}
-
-	return true
-}

go/weed/fix.go

@@ -1,70 +0,0 @@
-package main
-
-import (
-	"os"
-	"path"
-	"strconv"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/storage"
-)
-
-func init() {
-	cmdFix.Run = runFix // break init cycle
-}
-
-var cmdFix = &Command{
-	UsageLine: "fix -dir=/tmp -volumeId=234",
-	Short:     "run weed tool fix on index file if corrupted",
-	Long: `Fix runs the SeeweedFS fix command to re-create the index .idx file.
-
-  `,
-}
-
-var (
-	fixVolumePath       = cmdFix.Flag.String("dir", "/tmp", "data directory to store files")
-	fixVolumeCollection = cmdFix.Flag.String("collection", "", "the volume collection name")
-	fixVolumeId         = cmdFix.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
-)
-
-func runFix(cmd *Command, args []string) bool {
-
-	if *fixVolumeId == -1 {
-		return false
-	}
-
-	fileName := strconv.Itoa(*fixVolumeId)
-	if *fixVolumeCollection != "" {
-		fileName = *fixVolumeCollection + "_" + fileName
-	}
-	indexFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
-	if err != nil {
-		glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
-	}
-	defer indexFile.Close()
-
-	nm := storage.NewNeedleMap(indexFile)
-	defer nm.Close()
-
-	vid := storage.VolumeId(*fixVolumeId)
-	err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid,
-		storage.NeedleMapInMemory,
-		func(superBlock storage.SuperBlock) error {
-			return nil
-		}, false, func(n *storage.Needle, offset int64) error {
-			glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped())
-			if n.Size > 0 {
-				pe := nm.Put(n.Id, uint32(offset/storage.NeedlePaddingSize), n.Size)
-				glog.V(2).Infof("saved %d with error %v", n.Size, pe)
-			} else {
-				glog.V(2).Infof("skipping deleted file ...")
-				return nm.Delete(n.Id)
-			}
-			return nil
-		})
-	if err != nil {
-		glog.Fatalf("Export Volume File [ERROR] %s\n", err)
-	}
-
-	return true
-}

go/weed/master.go

@@ -1,95 +0,0 @@
-package main
-
-import (
-	"net/http"
-	"os"
-	"runtime"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
-	"github.com/chrislusf/weed-fs/go/weed/weed_server"
-	"github.com/gorilla/mux"
-)
-
-func init() {
-	cmdMaster.Run = runMaster // break init cycle
-}
-
-var cmdMaster = &Command{
-	UsageLine: "master -port=9333",
-	Short:     "start a master server",
-	Long: `start a master server to provide volume=>location mapping service
-  and sequence number of file ids
-
-  `,
-}
-
-var (
-	mport                   = cmdMaster.Flag.Int("port", 9333, "http listen port")
-	masterIp                = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address")
-	masterBindIp            = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
-	mPublicUrl              = cmdMaster.Flag.String("publicUrl", "", "peer accessible <ip>|<server_name>:port")
-	metaFolder              = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
-	masterPeers             = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list")
-	volumeSizeLimitMB       = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
-	mpulse                  = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
-	confFile                = cmdMaster.Flag.String("conf", "/etc/weedfs/weedfs.conf", "Deprecating! xml configuration file")
-	defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
-	mTimeout                = cmdMaster.Flag.Int("idleTimeout", 10, "connection idle seconds")
-	mMaxCpu                 = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
-	garbageThreshold        = cmdMaster.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
-	masterWhiteListOption   = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
-	masterSecureKey         = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
-
-	masterWhiteList []string
-)
-
-func runMaster(cmd *Command, args []string) bool {
-	if *mMaxCpu < 1 {
-		*mMaxCpu = runtime.NumCPU()
-	}
-	runtime.GOMAXPROCS(*mMaxCpu)
-	if err := util.TestFolderWritable(*metaFolder); err != nil {
-		glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err)
-	}
-	if *masterWhiteListOption != "" {
-		masterWhiteList = strings.Split(*masterWhiteListOption, ",")
-	}
-
-	r := mux.NewRouter()
-	ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
-		*volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold,
-		masterWhiteList, *masterSecureKey,
-	)
-
-	listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport)
-
-	glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)
-
-	listener, e := util.NewListener(listeningAddress, time.Duration(*mTimeout)*time.Second)
-	if e != nil {
-		glog.Fatalf("Master startup error: %v", e)
-	}
-
-	go func() {
-		time.Sleep(100 * time.Millisecond)
-		myMasterAddress := *masterIp + ":" + strconv.Itoa(*mport)
-		if *mPublicUrl != "" {
-			myMasterAddress = *mPublicUrl
-		}
-		var peers []string
-		if *masterPeers != "" {
-			peers = strings.Split(*masterPeers, ",")
-		}
-		raftServer := weed_server.NewRaftServer(r, peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
-		ms.SetRaftServer(raftServer)
-	}()
-
-	if e := http.Serve(listener, r); e != nil {
-		glog.Fatalf("Fail to serve: %v", e)
-	}
-	return true
-}

go/weed/mount.go

@@ -1,35 +0,0 @@
-package main
-
-type MountOptions struct {
-	filer *string
-	dir   *string
-}
-
-var (
-	mountOptions MountOptions
-)
-
-func init() {
-	cmdMount.Run = runMount // break init cycle
-	cmdMount.IsDebug = cmdMount.Flag.Bool("debug", false, "verbose debug information")
-	mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weedfs filer location")
-	mountOptions.dir = cmdMount.Flag.String("dir", "", "mount weed filer to this directory")
-}
-
-var cmdMount = &Command{
-	UsageLine: "mount -filer=localhost:8888 -dir=/some/dir",
-	Short:     "mount weed filer to a directory as file system in userspace(FUSE)",
-	Long: `mount weed file system to userspace.
-  
-  Pre-requisites:
-  1) have a weed file system running
-  2) have a "weed filer" running
-  These 2 requirements can be achieved with one command "weed server -filer=true"
-
-  This uses bazil.org/fuse, whichenables writing FUSE file systems on
-  Linux, and OS X.
-
-  On OS X, it requires OSXFUSE (http://osxfuse.github.com/).
-
-  `,
-}

go/weed/mount_std.go

@@ -1,104 +0,0 @@
-// +build linux darwin
-
-package main
-
-import (
-	"fmt"
-	"runtime"
-
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
-	"github.com/chrislusf/weed-fs/go/filer"
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/storage"
-	"github.com/chrislusf/weed-fs/go/util"
-	"golang.org/x/net/context"
-)
-
-func runMount(cmd *Command, args []string) bool {
-	fmt.Printf("This is Seaweed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
-	if *mountOptions.dir == "" {
-		fmt.Printf("Please specify the mount directory via \"-dir\"")
-		return false
-	}
-
-	c, err := fuse.Mount(*mountOptions.dir)
-	if err != nil {
-		glog.Fatal(err)
-		return false
-	}
-
-	OnInterrupt(func() {
-		fuse.Unmount(*mountOptions.dir)
-		c.Close()
-	})
-
-	err = fs.Serve(c, WFS{})
-	if err != nil {
-		fuse.Unmount(*mountOptions.dir)
-	}
-
-	// check if the mount process has an error to report
-	<-c.Ready
-	if err := c.MountError; err != nil {
-		glog.Fatal(err)
-	}
-
-	return true
-}
-
-type File struct {
-	FileId filer.FileId
-	Name   string
-}
-
-func (File) Attr(attr *fuse.Attr) {
-}
-func (File) ReadAll(ctx context.Context) ([]byte, error) {
-	return []byte("hello, world\n"), nil
-}
-
-type Dir struct {
-	Path string
-	Id   uint64
-}
-
-func (dir Dir) Attr(attr *fuse.Attr) {
-}
-
-func (dir Dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
-	files_result, e := filer.ListFiles(*mountOptions.filer, dir.Path, name)
-	if e != nil {
-		return nil, fuse.ENOENT
-	}
-	if len(files_result.Files) > 0 {
-		return File{files_result.Files[0].Id, files_result.Files[0].Name}, nil
-	}
-	return nil, fmt.Errorf("File Not Found for %s", name)
-}
-
-type WFS struct{}
-
-func (WFS) Root() (fs.Node, error) {
-	return Dir{}, nil
-}
-
-func (dir *Dir) ReadDir(ctx context.Context) ([]fuse.Dirent, error) {
-	var ret []fuse.Dirent
-	if dirs, e := filer.ListDirectories(*mountOptions.filer, dir.Path); e == nil {
-		for _, d := range dirs.Directories {
-			dirId := uint64(d.Id)
-			ret = append(ret, fuse.Dirent{Inode: dirId, Name: d.Name, Type: fuse.DT_Dir})
-		}
-	}
-	if files, e := filer.ListFiles(*mountOptions.filer, dir.Path, ""); e == nil {
-		for _, f := range files.Files {
-			if fileId, e := storage.ParseFileId(string(f.Id)); e == nil {
-				fileInode := uint64(fileId.VolumeId)<<48 + fileId.Key
-				ret = append(ret, fuse.Dirent{Inode: fileInode, Name: f.Name, Type: fuse.DT_File})
-			}
-
-		}
-	}
-	return ret, nil
-}