Update release_binaries.yml

fix https://github.com/chrislusf/seaweedfs/issues/2260

.github/workflows/release.yml

@@ -30,7 +30,7 @@ jobs:
           time: '30s'
 
       - name: Set BUILD_TIME env
-        run: echo BUILD_TIME=$(date -u +%Y-%m-%d-%H-%M) >> ${GITHUB_ENV}
+        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries
         uses: wangyoucao577/go-release-action@v1.17

.github/workflows/release_binaries.yml

@@ -0,0 +1,59 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Build Versioned Releases
+
+on:
+  release:
+    types: [created]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, windows, darwin, freebsd ]
+        goarch: [amd64, arm]
+        exclude:
+          - goarch: arm
+            goos: darwin
+          - goarch: arm
+            goos: windows
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - name: Go Release Binaries
+        uses: wangyoucao577/go-release-action@v1.19
+        with:
+          goversion: 1.16
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          # build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
+      - name: Go Release Large Disk Binaries
+        uses: wangyoucao577/go-release-action@v1.19
+        with:
+          goversion: 1.16
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"

README.md

@@ -69,16 +69,16 @@ Table of Contents
 * [License](#license)
 
 
+## Quick Start for S3 API on Docker ##
+
+`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
+
 ## Quick Start with single binary ##
 * Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
 * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
 
 Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
 
-## Quick Start for S3 API on Docker ##
-
-`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
-
 ## Introduction ##
 
 SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
@@ -146,6 +146,7 @@ Faster and Cheaper than direct cloud storage!
 * [WebDAV] accesses as a mapped drive on Mac and Windows, or from mobile devices.
 * [AES256-GCM Encrypted Storage][FilerDataEncryption] safely stores the encrypted data.
 * [Super Large Files][SuperLargeFiles] stores large or super large files in tens of TB.
+* [Cloud Data Accelerator][RemoteStorage] transparently read and write existing cloud data at local speed with content cache, metadata cache, and asynchronous write back.
 
 ## Kubernetes ##
 * [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
@@ -168,6 +169,8 @@ Faster and Cheaper than direct cloud storage!
 [ActiveActiveAsyncReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization
 [FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication
 [KeyLargeValueStore]: https://github.com/chrislusf/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
+[RemoteStorage]: https://github.com/chrislusf/seaweedfs/wiki/Remote-Storage-Architecture
+
 
 [Back to TOC](#table-of-contents)
 
@@ -321,7 +324,7 @@ When requesting a file key, an optional "dataCenter" parameter can limit the ass
 
 [Back to TOC](#table-of-contents)
 
-## Architecture ##
+## Object Store Architecture ##
 
 Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.
 

docker/Makefile

@@ -25,6 +25,9 @@ dev_tls: build certstrap
 dev_mount: build
 	docker-compose -f compose/local-mount-compose.yml -p seaweedfs up
 
+run_image: build
+	docker run --rm -ti --device /dev/fuse --cap-add SYS_ADMIN --entrypoint /bin/sh chrislusf/seaweedfs:local
+
 profile_mount: build
 	docker-compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
 

go.mod

@@ -14,7 +14,6 @@ require (
 	github.com/cespare/xxhash v1.1.0
 	github.com/chrislusf/raft v1.0.7
 	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/disintegration/imaging v1.6.2
 	github.com/dustin/go-humanize v1.0.0
 	github.com/eapache/go-resiliency v1.2.0 // indirect
@@ -28,10 +27,12 @@ require (
 	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-redis/redis/v8 v8.4.4
 	github.com/go-sql-driver/mysql v1.5.0
-	github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6
+	github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
 	github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect
+	github.com/golang-jwt/jwt v3.2.1+incompatible
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e
 	github.com/golang/protobuf v1.4.3
+	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.0.0
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.4
@@ -53,10 +54,11 @@ require (
 	github.com/olivere/elastic/v7 v7.0.19
 	github.com/peterh/liner v1.1.0
 	github.com/pierrec/lz4 v2.2.7+incompatible // indirect
+	github.com/posener/complete v1.2.3
 	github.com/pquerna/cachecontrol v0.1.0
 	github.com/prometheus/client_golang v1.11.0
 	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
-	github.com/seaweedfs/fuse v1.1.7
+	github.com/seaweedfs/fuse v1.1.8
 	github.com/seaweedfs/goexif v1.0.2
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
@@ -66,8 +68,8 @@ require (
 	github.com/stretchr/testify v1.6.1
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
-	github.com/tidwall/gjson v1.3.2
-	github.com/tidwall/match v1.0.1
+	github.com/tidwall/gjson v1.8.1
+	github.com/tidwall/match v1.0.3
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/viant/assertly v0.5.4 // indirect
@@ -76,7 +78,7 @@ require (
 	github.com/willf/bitset v1.1.10 // indirect
 	github.com/willf/bloom v2.0.3+incompatible
 	go.etcd.io/etcd v3.3.15+incompatible
-	go.mongodb.org/mongo-driver v1.3.2
+	go.mongodb.org/mongo-driver v1.7.0
 	gocloud.dev v0.20.0
 	gocloud.dev/pubsub/natspubsub v0.20.0
 	gocloud.dev/pubsub/rabbitpubsub v0.20.0

go.sum

@@ -258,6 +258,8 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6 h1:P66kRWyEoIx6URKgAC3ijx9jo9gEid7bEhLQ/Z0G65A=
 github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6/go.mod h1:Q7Sru5153KG8D9zwueuQJB3ccJf9/bIwF/x8b3oKgT8=
+github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d h1:k544nNVphXK4Yt0FTduvOvCfJabEY/DMkdNw0zpCwBE=
+github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -265,6 +267,8 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 h1:X+zN6RZXsvnrSJaAIQhZezPfAfvsqihKKR8oiLHid34=
 github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
+github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -298,6 +302,9 @@ github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8l
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -307,6 +314,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
@@ -361,10 +369,12 @@ github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
 github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
@@ -549,8 +559,8 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=
-github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI=
+github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
 github.com/peterh/liner v1.1.0 h1:f+aAedNJA6uk7+6rXsYBnhdo4Xux7ESLe+kcuVUF5os=
 github.com/peterh/liner v1.1.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
@@ -568,6 +578,8 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo=
+github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
 github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8cTqKc=
 github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -615,8 +627,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/seaweedfs/fuse v1.1.7 h1:T4L5c/Sn+q8lE+0zCmH2MWvIO+B5TttWOSqK5KQPRMQ=
-github.com/seaweedfs/fuse v1.1.7/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
+github.com/seaweedfs/fuse v1.1.8 h1:YFSDPotG4uhQzV7ooDUvQ8BRVy5rM1XCFPJAmAsZz68=
+github.com/seaweedfs/fuse v1.1.8/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
 github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E=
 github.com/seaweedfs/goexif v1.0.2/go.mod h1:MrKs5LK0HXdffrdCZrW3OIMegL2xXpC6ThLyXMyjdrk=
 github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a h1:C6IhVTxNkhlb0tlCB6JfHOUv1f0xHPK7V8X4HlJZEJw=
@@ -675,12 +687,13 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok=
 github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8=
-github.com/tidwall/gjson v1.3.2 h1:+7p3qQFaH3fOMXAJSrdZwGKcOO/lYdGS0HqGhPqDdTI=
-github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
-github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
-github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
-github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/gjson v1.8.1 h1:8j5EE9Hrh3l9Od1OIEDAb7IpezNA20UdRngNAj5N0WU=
+github.com/tidwall/gjson v1.8.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
+github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
+github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
+github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -703,14 +716,19 @@ github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX
 github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8=
 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
-github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w=
+github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
+github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc=
+github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
-github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
-github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -719,8 +737,8 @@ go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
 go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547 h1:s71VGheLtWmCYsnNjf+s7XE8HsrZnd3EYGrLGWVm7nY=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547/go.mod h1:YoUyTScD3Vcv2RBm3eGVOq7i1ULiz3OuXoQFWOirmAM=
-go.mongodb.org/mongo-driver v1.3.2 h1:IYppNjEV/C+/3VPbhHVxQ4t04eVW0cLp0/pNdW++6Ug=
-go.mongodb.org/mongo-driver v1.3.2/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
+go.mongodb.org/mongo-driver v1.7.0 h1:hHrvOBWlWB2c7+8Gh/Xi5jj82AgidK/t7KVXBZ+IyUA=
+go.mongodb.org/mongo-driver v1.7.0/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
 go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
@@ -766,6 +784,7 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
@@ -919,8 +938,9 @@ golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fq
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

k8s/seaweedfs/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.56"
-version: "2.56"
+appVersion: "2.63"
+version: "2.63"

k8s/seaweedfs/templates/volume-statefulset.yaml

@@ -110,7 +110,7 @@ spec:
                 -dataCenter={{ .Values.volume.dataCenter }} \
                 {{- end }}
                 -ip.bind={{ .Values.volume.ipBind }} \
-                -read.redirect={{ .Values.volume.readRedirect }} \
+                -readMode={{ .Values.volume.readMode }} \
                 {{- if .Values.volume.whiteList }}
                 -whiteList={{ .Values.volume.whiteList }} \
                 {{- end }}

k8s/seaweedfs/values.yaml

@@ -4,7 +4,6 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  # imageTag: "2.56" - started using {.Chart.appVersion}
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always
@@ -175,8 +174,8 @@ volume:
   # Volume server's data center name
   dataCenter: null
 
-  # Redirect moved or non-local volumes. (default true)
-  readRedirect: true
+  # Redirect moved or non-local volumes. (default proxy)
+  readMode: proxy
 
   # Comma separated Ip addresses having write permission. No limit if empty.
   whiteList: null

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.6</version>
+    <version>1.6.7</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/pom.xml.deploy

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.6</version>
+    <version>1.6.7</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/pom_debug.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.6</version>
+    <version>1.6.7</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java

@@ -10,7 +10,7 @@ import java.util.List;
 
 public class ByteBufferPool {
 
-    private static final int MIN_BUFFER_SIZE = 8 * 1024 * 1024;
+    private static final int MIN_BUFFER_SIZE = 1 * 1024 * 1024;
     private static final Logger LOG = LoggerFactory.getLogger(ByteBufferPool.class);
 
     private static final List<ByteBuffer> bufferList = new ArrayList<>();

other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java

@@ -0,0 +1,23 @@
+package seaweedfs.client;
+
+import java.io.IOException;
+
+public class RemoteUtil {
+    public static boolean isInRemoteOnly(FilerProto.Entry entry) {
+        if (entry.getChunksList() == null || entry.getChunksList().isEmpty()) {
+            return entry.getRemoteEntry() != null && entry.getRemoteEntry().getRemoteSize() > 0;
+        }
+        return false;
+    }
+
+    public static FilerProto.Entry downloadRemoteEntry(FilerClient filerClient, String fullpath, FilerProto.Entry entry) throws IOException {
+        String dir = SeaweedOutputStream.getParentDirectory(fullpath);
+        String name = SeaweedOutputStream.getFileName(fullpath);
+
+        final FilerProto.DownloadToLocalResponse downloadToLocalResponse = filerClient.getBlockingStub()
+                .downloadToLocal(FilerProto.DownloadToLocalRequest.newBuilder()
+                .setDirectory(dir).setName(name).build());
+
+        return downloadToLocalResponse.getEntry();
+    }
+}

other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java

@@ -19,9 +19,9 @@ public class SeaweedInputStream extends InputStream {
 
     private final FilerClient filerClient;
     private final String path;
-    private final FilerProto.Entry entry;
     private final List<SeaweedRead.VisibleInterval> visibleIntervalList;
     private final long contentLength;
+    private FilerProto.Entry entry;
 
     private long position = 0;  // cursor of the file
 
@@ -35,10 +35,14 @@ public class SeaweedInputStream extends InputStream {
         this.entry = filerClient.lookupEntry(
                 SeaweedOutputStream.getParentDirectory(fullpath),
                 SeaweedOutputStream.getFileName(fullpath));
-        if(entry == null){
+        if (entry == null) {
             throw new FileNotFoundException();
         }
 
+        if (RemoteUtil.isInRemoteOnly(entry)) {
+            entry = RemoteUtil.downloadRemoteEntry(filerClient, fullpath, entry);
+        }
+
         this.contentLength = SeaweedRead.fileSize(entry);
 
         this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList());
@@ -54,6 +58,11 @@ public class SeaweedInputStream extends InputStream {
         this.filerClient = filerClient;
         this.path = path;
         this.entry = entry;
+
+        if (RemoteUtil.isInRemoteOnly(entry)) {
+            this.entry = RemoteUtil.downloadRemoteEntry(filerClient, path, entry);
+        }
+
         this.contentLength = SeaweedRead.fileSize(entry);
 
         this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList());
@@ -111,8 +120,8 @@ public class SeaweedInputStream extends InputStream {
         long bytesRead = 0;
         int len = buf.remaining();
         int start = (int) this.position;
-        if (start+len <= entry.getContent().size()) {
-            entry.getContent().substring(start, start+len).copyTo(buf);
+        if (start + len <= entry.getContent().size()) {
+            entry.getContent().substring(start, start + len).copyTo(buf);
         } else {
             bytesRead = SeaweedRead.read(this.filerClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry));
         }

other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java

@@ -32,10 +32,10 @@ public class SeaweedOutputStream extends OutputStream {
     private long lastTotalAppendOffset = 0;
     private ByteBuffer buffer;
     private long outputIndex;
-    private String replication = "000";
+    private String replication = "";
 
     public SeaweedOutputStream(FilerClient filerClient, final String fullpath) {
-        this(filerClient, fullpath, "000");
+        this(filerClient, fullpath, "");
     }
 
     public SeaweedOutputStream(FilerClient filerClient, final String fullpath, final String replication) {

other/java/client/src/main/proto/filer.proto

@@ -67,6 +67,8 @@ service SeaweedFiler {
     rpc KvPut (KvPutRequest) returns (KvPutResponse) {
     }
 
+    rpc DownloadToLocal (DownloadToLocalRequest) returns (DownloadToLocalResponse) {
+    }
 }
 
 //////////////////////////////////////////////////
@@ -92,6 +94,13 @@ message ListEntriesResponse {
     Entry entry = 1;
 }
 
+message RemoteEntry {
+    string storage_name = 1;
+    int64 last_local_sync_ts_ns = 2;
+    string remote_e_tag = 3;
+    int64 remote_mtime = 4;
+    int64 remote_size = 5;
+}
 message Entry {
     string name = 1;
     bool is_directory = 2;
@@ -101,6 +110,8 @@ message Entry {
     bytes hard_link_id = 7;
     int32 hard_link_counter = 8; // only exists in hard link meta data
     bytes content = 9; // if not empty, the file content
+
+    RemoteEntry remote_entry = 10;
 }
 
 message FullEntry {
@@ -208,6 +219,7 @@ message AtomicRenameEntryRequest {
     string old_name = 2;
     string new_directory = 3;
     string new_name = 4;
+    repeated int32 signatures = 5;
 }
 
 message AtomicRenameEntryResponse {
@@ -292,6 +304,7 @@ message GetFilerConfigurationResponse {
     int32 signature = 8;
     string metrics_address = 9;
     int32 metrics_interval_sec = 10;
+    string version = 11;
 }
 
 message SubscribeMetadataRequest {
@@ -334,7 +347,9 @@ message LocateBrokerResponse {
     repeated Resource resources = 2;
 }
 
+/////////////////////////
 // Key-Value operations
+/////////////////////////
 message KvGetRequest {
     bytes key = 1;
 }
@@ -350,7 +365,9 @@ message KvPutResponse {
     string error = 1;
 }
 
+/////////////////////////
 // path-based configurations
+/////////////////////////
 message FilerConf {
     int32 version = 1;
     message PathConf {
@@ -365,3 +382,31 @@ message FilerConf {
     }
     repeated PathConf locations = 2;
 }
+
+/////////////////////////
+// Remote Storage related
+/////////////////////////
+message RemoteConf {
+    string type = 1;
+    string name = 2;
+    string s3_access_key = 4;
+    string s3_secret_key = 5;
+    string s3_region = 6;
+    string s3_endpoint = 7;
+}
+
+message RemoteStorageMapping {
+    map<string,RemoteStorageLocation> mappings = 1;
+}
+message RemoteStorageLocation {
+    string name = 1;
+    string bucket = 2;
+    string path = 3;
+}
+message DownloadToLocalRequest {
+    string directory = 1;
+    string name = 2;
+}
+message DownloadToLocalResponse {
+    Entry entry = 1;
+}

other/java/examples/pom.xml

@@ -11,13 +11,13 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>1.6.6</version>
+            <version>1.6.7</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>1.6.6</version>
+            <version>1.6.7</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

other/java/hdfs2/dependency-reduced-pom.xml

@@ -301,7 +301,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.6.6</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
     <hadoop.version>2.9.2</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.6</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
         <hadoop.version>2.9.2</hadoop.version>
     </properties>
 

other/java/hdfs3/dependency-reduced-pom.xml

@@ -309,7 +309,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.6.6</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
     <hadoop.version>3.1.1</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.6</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 

unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go

@@ -1,14 +1,12 @@
 package main
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"google.golang.org/grpc"
-	"io"
 	"strconv"
 	"time"
 )
@@ -74,38 +72,9 @@ func startGenerateMetadata() {
 
 func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
 
-	lastTsNs := int64(0)
+	tailErr := pb.FollowMetadata(*tailFiler, grpc.WithInsecure(),	"tail",
+		*dir, 0, 0, eachEntryFunc, false)
 
-	tailErr := pb.WithFilerClient(*tailFiler, grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error {
-
-		ctx, cancel := context.WithCancel(context.Background())
-		defer cancel()
-
-		stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
-			ClientName: "tail",
-			PathPrefix: *dir,
-			SinceNs:    lastTsNs,
-		})
-		if err != nil {
-			return fmt.Errorf("listen: %v", err)
-		}
-
-		for {
-			resp, listenErr := stream.Recv()
-			if listenErr == io.EOF {
-				return nil
-			}
-			if listenErr != nil {
-				return listenErr
-			}
-			if err = eachEntryFunc(resp); err != nil {
-				glog.V(0).Infof("tail last record:%+v", time.Unix(0, lastTsNs))
-				return err
-			}
-			lastTsNs = resp.TsNs
-		}
-
-	})
 	if tailErr != nil {
 		fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
 	}

unmaintained/see_meta/see_meta.go

@@ -60,7 +60,7 @@ func walkMetaFile(dst *os.File) error {
 
 		fmt.Fprintf(os.Stdout, "file %s %v\n", util.FullPath(fullEntry.Dir).Child(fullEntry.Entry.Name), fullEntry.Entry.Attributes.String())
 		for i, chunk := range fullEntry.Entry.Chunks {
-			fmt.Fprintf(os.Stdout, "  chunk %d %v\n", i+1, chunk.String())
+			fmt.Fprintf(os.Stdout, "  chunk: %d %v %d,%x%08x\n", i+1, chunk, chunk.Fid.VolumeId, chunk.Fid.FileKey, chunk.Fid.Cookie)
 		}
 
 	}

weed/Makefile

@@ -16,7 +16,7 @@ debug_shell:
 
 debug_mount:
 	go build -gcflags="all=-N -l"
-	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/buckets
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/
 
 debug_server:
 	go build -gcflags="all=-N -l"
@@ -37,3 +37,12 @@ debug_s3:
 debug_filer_copy:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.backup -filer=localhost:8888 -filerProxy -timeAgo=10h
+
+debug_filer_remote_sync:
+	go build -gcflags="all=-N -l"
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 filer.remote.sync -filer="localhost:8888" -dir=/buckets/b2 -timeAgo=1h
+
+debug_master_follower:
+	go build -gcflags="all=-N -l"
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 master.follower
+

weed/command/autocomplete.go

@@ -0,0 +1,109 @@
+package command
+
+import (
+	"fmt"
+	flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
+	"github.com/posener/complete"
+	completeinstall "github.com/posener/complete/cmd/install"
+	"runtime"
+)
+
+func AutocompleteMain(commands []*Command) bool {
+	subCommands := make(map[string]complete.Command)
+	helpSubCommands := make(map[string]complete.Command)
+	for _, cmd := range commands {
+		flags := make(map[string]complete.Predictor)
+		cmd.Flag.VisitAll(func(flag *flag.Flag) {
+			flags["-"+flag.Name] = complete.PredictAnything
+		})
+
+		subCommands[cmd.Name()] = complete.Command{
+			Flags: flags,
+		}
+		helpSubCommands[cmd.Name()] = complete.Command{}
+	}
+	subCommands["help"] = complete.Command{Sub: helpSubCommands}
+
+	globalFlags := make(map[string]complete.Predictor)
+	flag.VisitAll(func(flag *flag.Flag) {
+		globalFlags["-"+flag.Name] = complete.PredictAnything
+	})
+
+	weedCmd := complete.Command{
+		Sub:         subCommands,
+		Flags:       globalFlags,
+		GlobalFlags: complete.Flags{"-h": complete.PredictNothing},
+	}
+	cmp := complete.New("weed", weedCmd)
+
+	return cmp.Complete()
+}
+
+func installAutoCompletion() bool {
+	if runtime.GOOS == "windows" {
+		fmt.Println("windows is not supported")
+		return false
+	}
+
+	err := completeinstall.Install("weed")
+	if err != nil {
+		fmt.Printf("install failed! %s\n", err)
+		return false
+	}
+	fmt.Printf("autocompletion is enabled. Please restart your shell.\n")
+	return true
+}
+
+func uninstallAutoCompletion() bool {
+	if runtime.GOOS == "windows" {
+		fmt.Println("windows is not supported")
+		return false
+	}
+
+	err := completeinstall.Uninstall("weed")
+	if err != nil {
+		fmt.Printf("uninstall failed! %s\n", err)
+		return false
+	}
+	fmt.Printf("autocompletion is disable. Please restart your shell.\n")
+	return true
+}
+
+var cmdAutocomplete = &Command{
+	Run:       runAutocomplete,
+	UsageLine: "autocomplete",
+	Short:     "install autocomplete",
+	Long: `weed autocomplete is installed in the shell.
+
+    Supported shells are bash, zsh, and fish.
+    Windows is not supported.
+
+`,
+}
+
+func runAutocomplete(cmd *Command, args []string) bool {
+	if len(args) != 0 {
+		cmd.Usage()
+	}
+
+	return installAutoCompletion()
+}
+
+var cmdUnautocomplete = &Command{
+	Run:       runUnautocomplete,
+	UsageLine: "autocomplete.uninstall",
+	Short:     "uninstall autocomplete",
+	Long: `weed autocomplete is uninstalled in the shell.
+
+    Windows is not supported.
+
+`,
+}
+
+func runUnautocomplete(cmd *Command, args []string) bool {
+	if len(args) != 0 {
+		cmd.Usage()
+	}
+
+	return uninstallAutoCompletion()
+}

weed/command/backup.go

@@ -72,7 +72,7 @@ func runBackup(cmd *Command, args []string) bool {
 	vid := needle.VolumeId(*s.volumeId)
 
 	// find volume location, replication, ttl info
-	lookup, err := operation.Lookup(func() string { return *s.master }, vid.String())
+	lookup, err := operation.LookupVolumeId(func() string { return *s.master }, grpcDialOption, vid.String())
 	if err != nil {
 		fmt.Printf("Error looking up volume %d: %v\n", vid, err)
 		return true

weed/command/benchmark.go

@@ -212,7 +212,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 				}
 				var jwtAuthorization security.EncodedJwt
 				if isSecure {
-					jwtAuthorization = operation.LookupJwt(b.masterClient.GetMaster(), df.fp.Fid)
+					jwtAuthorization = operation.LookupJwt(b.masterClient.GetMaster(), b.grpcDialOption, df.fp.Fid)
 				}
 				if e := util.Delete(fmt.Sprintf("http://%s/%s", df.fp.Server, df.fp.Fid), string(jwtAuthorization)); e == nil {
 					s.completed++

weed/command/command.go

@@ -8,23 +8,26 @@ import (
 )
 
 var Commands = []*Command{
-	cmdBenchmark,
+	cmdAutocomplete,
+	cmdUnautocomplete,
 	cmdBackup,
+	cmdBenchmark,
 	cmdCompact,
-	cmdCopy,
 	cmdDownload,
 	cmdExport,
 	cmdFiler,
 	cmdFilerBackup,
 	cmdFilerCat,
+	cmdFilerCopy,
 	cmdFilerMetaBackup,
 	cmdFilerMetaTail,
+	cmdFilerRemoteSynchronize,
 	cmdFilerReplicate,
 	cmdFilerSynchronize,
 	cmdFix,
 	cmdFuse,
-	cmdGateway,
 	cmdMaster,
+	cmdMasterFollower,
 	cmdMount,
 	cmdS3,
 	cmdIam,

weed/command/download.go

@@ -2,6 +2,8 @@ package command
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"google.golang.org/grpc"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -43,20 +45,23 @@ var cmdDownload = &Command{
 }
 
 func runDownload(cmd *Command, args []string) bool {
+	util.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
 	for _, fid := range args {
-		if e := downloadToFile(func() string { return *d.server }, fid, util.ResolvePath(*d.dir)); e != nil {
+		if e := downloadToFile(func() string { return *d.server }, grpcDialOption, fid, util.ResolvePath(*d.dir)); e != nil {
 			fmt.Println("Download Error: ", fid, e)
 		}
 	}
 	return true
 }
 
-func downloadToFile(masterFn operation.GetMasterFn, fileId, saveDir string) error {
-	fileUrl, lookupError := operation.LookupFileId(masterFn, fileId)
+func downloadToFile(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption, fileId, saveDir string) error {
+	fileUrl, jwt, lookupError := operation.LookupFileId(masterFn, grpcDialOption, fileId)
 	if lookupError != nil {
 		return lookupError
 	}
-	filename, _, rc, err := util.DownloadFile(fileUrl)
+	filename, _, rc, err := util.DownloadFile(fileUrl, jwt)
 	if err != nil {
 		return err
 	}
@@ -83,7 +88,7 @@ func downloadToFile(masterFn operation.GetMasterFn, fileId, saveDir string) erro
 		fids := strings.Split(string(content), "\n")
 		for _, partId := range fids {
 			var n int
-			_, part, err := fetchContent(masterFn, partId)
+			_, part, err := fetchContent(masterFn, grpcDialOption, partId)
 			if err == nil {
 				n, err = f.Write(part)
 			}
@@ -103,13 +108,13 @@ func downloadToFile(masterFn operation.GetMasterFn, fileId, saveDir string) erro
 	return nil
 }
 
-func fetchContent(masterFn operation.GetMasterFn, fileId string) (filename string, content []byte, e error) {
-	fileUrl, lookupError := operation.LookupFileId(masterFn, fileId)
+func fetchContent(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption, fileId string) (filename string, content []byte, e error) {
+	fileUrl, jwt, lookupError := operation.LookupFileId(masterFn, grpcDialOption, fileId)
 	if lookupError != nil {
 		return "", nil, lookupError
 	}
 	var rc *http.Response
-	if filename, _, rc, e = util.DownloadFile(fileUrl); e != nil {
+	if filename, _, rc, e = util.DownloadFile(fileUrl, jwt); e != nil {
 		return "", nil, e
 	}
 	defer util.CloseResponse(rc)

weed/command/filer.go

@@ -50,6 +50,8 @@ type FilerOptions struct {
 	saveToFilerLimit        *int
 	defaultLevelDbDirectory *string
 	concurrentUploadLimitMB *int
+	debug                   *bool
+	debugPort               *int
 }
 
 func init() {
@@ -73,6 +75,8 @@ func init() {
 	f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
 	f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
 	f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
+	f.debug = cmdFiler.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
+	f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging")
 
 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -122,6 +126,9 @@ var cmdFiler = &Command{
 }
 
 func runFiler(cmd *Command, args []string) bool {
+	if *f.debug {
+		go http.ListenAndServe(fmt.Sprintf(":%d", *f.debugPort), nil)
+	}
 
 	util.LoadConfiguration("security", false)
 

weed/command/filer_backup.go

@@ -1,16 +1,13 @@
 package command
 
 import (
-	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/replication/source"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"google.golang.org/grpc"
-	"io"
 	"time"
 )
 
@@ -52,11 +49,11 @@ var cmdFilerBackup = &Command{
 
 func runFilerBackup(cmd *Command, args []string) bool {
 
-	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
-
 	util.LoadConfiguration("security", false)
 	util.LoadConfiguration("replication", true)
 
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
 	for {
 		err := doFilerBackup(grpcDialOption, &filerBackupOptions)
 		if err != nil {
@@ -110,48 +107,12 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 
 	processEventFn := genProcessFunction(sourcePath, targetPath, dataSink, debug)
 
-	return pb.WithFilerClient(sourceFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-
-		ctx, cancel := context.WithCancel(context.Background())
-		defer cancel()
-
-		stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
-			ClientName: "backup_" + dataSink.GetName(),
-			PathPrefix: sourcePath,
-			SinceNs:    startFrom.UnixNano(),
-		})
-		if err != nil {
-			return fmt.Errorf("listen: %v", err)
-		}
-
-		var counter int64
-		var lastWriteTime time.Time
-		for {
-			resp, listenErr := stream.Recv()
-
-			if listenErr == io.EOF {
-				return nil
-			}
-			if listenErr != nil {
-				return listenErr
-			}
-
-			if err := processEventFn(resp); err != nil {
-				return fmt.Errorf("processEventFn: %v", err)
-			}
-
-			counter++
-			if lastWriteTime.Add(3 * time.Second).Before(time.Now()) {
-				glog.V(0).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, resp.TsNs), float64(counter)/float64(3))
-				counter = 0
-				lastWriteTime = time.Now()
-				if err := setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), resp.TsNs); err != nil {
-					return fmt.Errorf("setOffset: %v", err)
-				}
-			}
-
-		}
-
+	processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error {
+		glog.V(0).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, lastTsNs), float64(counter)/float64(3))
+		return setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), lastTsNs)
 	})
 
+	return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(),
+		sourcePath, startFrom.UnixNano(), 0, processEventFnWithOffset, false)
+
 }

weed/command/filer_copy.go

@@ -52,21 +52,21 @@ type CopyOptions struct {
 }
 
 func init() {
-	cmdCopy.Run = runCopy // break init cycle
-	cmdCopy.IsDebug = cmdCopy.Flag.Bool("debug", false, "verbose debug information")
-	copy.include = cmdCopy.Flag.String("include", "", "pattens of files to copy, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
-	copy.replication = cmdCopy.Flag.String("replication", "", "replication type")
-	copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
-	copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
-	copy.diskType = cmdCopy.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
-	copy.maxMB = cmdCopy.Flag.Int("maxMB", 4, "split files larger than the limit")
-	copy.concurrenctFiles = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
-	copy.concurrenctChunks = cmdCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
-	copy.checkSize = cmdCopy.Flag.Bool("check.size", false, "copy when the target file size is different from the source file")
-	copy.verbose = cmdCopy.Flag.Bool("verbose", false, "print out details during copying")
+	cmdFilerCopy.Run = runCopy // break init cycle
+	cmdFilerCopy.IsDebug = cmdFilerCopy.Flag.Bool("debug", false, "verbose debug information")
+	copy.include = cmdFilerCopy.Flag.String("include", "", "pattens of files to copy, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
+	copy.replication = cmdFilerCopy.Flag.String("replication", "", "replication type")
+	copy.collection = cmdFilerCopy.Flag.String("collection", "", "optional collection name")
+	copy.ttl = cmdFilerCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
+	copy.diskType = cmdFilerCopy.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
+	copy.maxMB = cmdFilerCopy.Flag.Int("maxMB", 4, "split files larger than the limit")
+	copy.concurrenctFiles = cmdFilerCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
+	copy.concurrenctChunks = cmdFilerCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
+	copy.checkSize = cmdFilerCopy.Flag.Bool("check.size", false, "copy when the target file size is different from the source file")
+	copy.verbose = cmdFilerCopy.Flag.Bool("verbose", false, "print out details during copying")
 }
 
-var cmdCopy = &Command{
+var cmdFilerCopy = &Command{
 	UsageLine: "filer.copy file_or_dir1 [file_or_dir2 file_or_dir3] http://localhost:8888/path/to/a/folder/",
 	Short:     "copy one or a list of files to a filer folder",
 	Long: `copy one or a list of files, or batch copy one whole folder recursively, to a filer folder
@@ -154,7 +154,7 @@ func runCopy(cmd *Command, args []string) bool {
 	}
 	copy.ttlSec = int32(ttl.Minutes()) * 60
 
-	if *cmdCopy.IsDebug {
+	if *cmdFilerCopy.IsDebug {
 		grace.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof")
 	}
 
@@ -381,6 +381,9 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 				if assignResult.Error != "" {
 					return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error)
 				}
+				if assignResult.Url == "" {
+					return fmt.Errorf("assign volume failure %v: %v", request, assignResult)
+				}
 				return nil
 			})
 		})

weed/command/filer_meta_backup.go

@@ -7,7 +7,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/spf13/viper"
 	"google.golang.org/grpc"
-	"io"
 	"reflect"
 	"time"
 
@@ -53,6 +52,7 @@ The backup writes to another filer store specified in a backup_filer.toml.
 
 func runFilerMetaBackup(cmd *Command, args []string) bool {
 
+	util.LoadConfiguration("security", false)
 	metaBackup.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	// load backup_filer.toml
@@ -189,48 +189,15 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
 		return nil
 	}
 
-	tailErr := pb.WithFilerClient(*metaBackup.filerAddress, metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-
-		ctx, cancel := context.WithCancel(context.Background())
-		defer cancel()
-
-		stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
-			ClientName: "meta_backup",
-			PathPrefix: *metaBackup.filerDirectory,
-			SinceNs:    startTime.UnixNano(),
-		})
-		if err != nil {
-			return fmt.Errorf("listen: %v", err)
-		}
-
-		var counter int64
-		var lastWriteTime time.Time
-		for {
-			resp, listenErr := stream.Recv()
-			if listenErr == io.EOF {
-				return nil
-			}
-			if listenErr != nil {
-				return listenErr
-			}
-			if err = eachEntryFunc(resp); err != nil {
-				return err
-			}
-
-			counter++
-			if lastWriteTime.Add(3 * time.Second).Before(time.Now()) {
-				glog.V(0).Infof("meta backup %s progressed to %v %0.2f/sec", *metaBackup.filerAddress, time.Unix(0, resp.TsNs), float64(counter)/float64(3))
-				counter = 0
-				lastWriteTime = time.Now()
-				if err2 := metaBackup.setOffset(lastWriteTime); err2 != nil {
-					return err2
-				}
-			}
-
-		}
-
+	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
+		lastTime := time.Unix(0, lastTsNs)
+		glog.V(0).Infof("meta backup %s progressed to %v %0.2f/sec", *metaBackup.filerAddress, lastTime, float64(counter)/float64(3))
+		return metaBackup.setOffset(lastTime)
 	})
-	return tailErr
+
+	return pb.FollowMetadata(*metaBackup.filerAddress, metaBackup.grpcDialOption, "meta_backup",
+		*metaBackup.filerDirectory, startTime.UnixNano(), 0, processEventFnWithOffset, false)
+
 }
 
 func (metaBackup *FilerMetaBackupOptions) getOffset() (lastWriteTime time.Time, err error) {

weed/command/filer_meta_tail.go

@@ -3,16 +3,15 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/golang/protobuf/jsonpb"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/olivere/elastic/v7"
-	"io"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -45,6 +44,7 @@ var (
 
 func runFilerMetaTail(cmd *Command, args []string) bool {
 
+	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	var filterFunc func(dir, fname string) bool
@@ -103,37 +103,18 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
 		}
 	}
 
-	tailErr := pb.WithFilerClient(*tailFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-
-		ctx, cancel := context.WithCancel(context.Background())
-		defer cancel()
-
-		stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
-			ClientName: "tail",
-			PathPrefix: *tailTarget,
-			SinceNs:    time.Now().Add(-*tailStart).UnixNano(),
-		})
-		if err != nil {
-			return fmt.Errorf("listen: %v", err)
-		}
-
-		for {
-			resp, listenErr := stream.Recv()
-			if listenErr == io.EOF {
+	tailErr := pb.FollowMetadata(*tailFiler, grpcDialOption, "tail",
+		*tailTarget, time.Now().Add(-*tailStart).UnixNano(), 0,
+		func(resp *filer_pb.SubscribeMetadataResponse) error {
+			if !shouldPrint(resp) {
 				return nil
 			}
-			if listenErr != nil {
-				return listenErr
-			}
-			if !shouldPrint(resp) {
-				continue
-			}
-			if err = eachEntryFunc(resp); err != nil {
+			if err := eachEntryFunc(resp); err != nil {
 				return err
 			}
-		}
+			return nil
+		}, false)
 
-	})
 	if tailErr != nil {
 		fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
 	}

weed/command/filer_remote_sync.go

@@ -0,0 +1,238 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+	"github.com/chrislusf/seaweedfs/weed/replication/source"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"time"
+)
+
+type RemoteSyncOptions struct {
+	filerAddress       *string
+	grpcDialOption     grpc.DialOption
+	readChunkFromFiler *bool
+	debug              *bool
+	timeAgo            *time.Duration
+	dir                *string
+}
+
+const (
+	RemoteSyncKeyPrefix = "remote.sync."
+)
+
+var _ = filer_pb.FilerClient(&RemoteSyncOptions{})
+
+func (option *RemoteSyncOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
+	return pb.WithFilerClient(*option.filerAddress, option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		return fn(client)
+	})
+}
+func (option *RemoteSyncOptions) AdjustedUrl(location *filer_pb.Location) string {
+	return location.Url
+}
+
+var (
+	remoteSyncOptions RemoteSyncOptions
+)
+
+func init() {
+	cmdFilerRemoteSynchronize.Run = runFilerRemoteSynchronize // break init cycle
+	remoteSyncOptions.filerAddress = cmdFilerRemoteSynchronize.Flag.String("filer", "localhost:8888", "filer of the SeaweedFS cluster")
+	remoteSyncOptions.dir = cmdFilerRemoteSynchronize.Flag.String("dir", "/", "a mounted directory on filer")
+	remoteSyncOptions.readChunkFromFiler = cmdFilerRemoteSynchronize.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers")
+	remoteSyncOptions.debug = cmdFilerRemoteSynchronize.Flag.Bool("debug", false, "debug mode to print out filer updated remote files")
+	remoteSyncOptions.timeAgo = cmdFilerRemoteSynchronize.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
+}
+
+var cmdFilerRemoteSynchronize = &Command{
+	UsageLine: "filer.remote.sync -filer=<filerHost>:<filerPort> -dir=/mount/s3_on_cloud",
+	Short:     "resumable continuously write back updates to remote storage if the directory is mounted to the remote storage",
+	Long: `resumable continuously write back updates to remote storage if the directory is mounted to the remote storage
+
+	filer.remote.sync listens on filer update events. 
+	If any mounted remote file is updated, it will fetch the updated content,
+	and write to the remote storage.
+`,
+}
+
+func runFilerRemoteSynchronize(cmd *Command, args []string) bool {
+
+	util.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+	remoteSyncOptions.grpcDialOption = grpcDialOption
+
+	dir := *remoteSyncOptions.dir
+	filerAddress := *remoteSyncOptions.filerAddress
+
+	// read filer remote storage mount mappings
+	_, _, remoteStorageMountLocation, storageConf, detectErr := filer.DetectMountInfo(grpcDialOption, filerAddress, dir)
+	if detectErr != nil {
+		fmt.Printf("read mount info: %v", detectErr)
+		return false
+	}
+
+	filerSource := &source.FilerSource{}
+	filerSource.DoInitialize(
+		filerAddress,
+		pb.ServerToGrpcAddress(filerAddress),
+		"/", // does not matter
+		*remoteSyncOptions.readChunkFromFiler,
+	)
+
+	fmt.Printf("synchronize %s to remote storage...\n", dir)
+	util.RetryForever("filer.remote.sync "+dir, func() error {
+		return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir, storageConf, remoteStorageMountLocation)
+	}, func(err error) bool {
+		if err != nil {
+			glog.Errorf("synchronize %s: %v", dir, err)
+		}
+		return true
+	})
+
+	return true
+}
+
+func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string, remoteStorage *filer_pb.RemoteConf, remoteStorageMountLocation *filer_pb.RemoteStorageLocation) error {
+
+	dirHash := util.HashStringToLong(mountedDir)
+
+	// 1. specified by timeAgo
+	// 2. last offset timestamp for this directory
+	// 3. directory creation time
+	var lastOffsetTs time.Time
+	if *option.timeAgo == 0 {
+		mountedDirEntry, err := filer_pb.GetEntry(option, util.FullPath(mountedDir))
+		if err != nil {
+			return fmt.Errorf("lookup %s: %v", mountedDir, err)
+		}
+
+		lastOffsetTsNs, err := getOffset(option.grpcDialOption, *option.filerAddress, RemoteSyncKeyPrefix, int32(dirHash))
+		if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 {
+			lastOffsetTs = time.Unix(0, lastOffsetTsNs)
+			glog.V(0).Infof("resume from %v", lastOffsetTs)
+		} else {
+			lastOffsetTs = time.Unix(mountedDirEntry.Attributes.Crtime, 0)
+		}
+	} else {
+		lastOffsetTs = time.Now().Add(-*option.timeAgo)
+	}
+
+	client, err := remote_storage.GetRemoteStorage(remoteStorage)
+	if err != nil {
+		return err
+	}
+
+	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		message := resp.EventNotification
+		if message.OldEntry == nil && message.NewEntry == nil {
+			return nil
+		}
+		if message.OldEntry == nil && message.NewEntry != nil {
+			if !filer.HasData(message.NewEntry) {
+				return nil
+			}
+			glog.V(2).Infof("create: %+v", resp)
+			if !shouldSendToRemote(message.NewEntry) {
+				glog.V(2).Infof("skipping creating: %+v", resp)
+				return nil
+			}
+			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
+			if message.NewEntry.IsDirectory {
+				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
+				return client.WriteDirectory(dest, message.NewEntry)
+			}
+			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
+			reader := filer.NewFileReader(filerSource, message.NewEntry)
+			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
+			if writeErr != nil {
+				return writeErr
+			}
+			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+		}
+		if message.OldEntry != nil && message.NewEntry == nil {
+			glog.V(2).Infof("delete: %+v", resp)
+			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
+			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
+			return client.DeleteFile(dest)
+		}
+		if message.OldEntry != nil && message.NewEntry != nil {
+			oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
+			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
+			if !shouldSendToRemote(message.NewEntry) {
+				glog.V(2).Infof("skipping updating: %+v", resp)
+				return nil
+			}
+			if message.NewEntry.IsDirectory {
+				return client.WriteDirectory(dest, message.NewEntry)
+			}
+			if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
+				if filer.IsSameData(message.OldEntry, message.NewEntry) {
+					glog.V(2).Infof("update meta: %+v", resp)
+					return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry)
+				}
+			}
+			glog.V(2).Infof("update: %+v", resp)
+			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
+			if err := client.DeleteFile(oldDest); err != nil {
+				return err
+			}
+			reader := filer.NewFileReader(filerSource, message.NewEntry)
+			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
+			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
+			if writeErr != nil {
+				return writeErr
+			}
+			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+		}
+
+		return nil
+	}
+
+	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
+		lastTime := time.Unix(0, lastTsNs)
+		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
+		return setOffset(option.grpcDialOption, *option.filerAddress, RemoteSyncKeyPrefix, int32(dirHash), lastTsNs)
+	})
+
+	return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption,
+		"filer.remote.sync", mountedDir, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
+}
+
+func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *filer_pb.RemoteStorageLocation) *filer_pb.RemoteStorageLocation {
+	source := string(sourcePath[len(mountDir):])
+	dest := util.FullPath(remoteMountLocation.Path).Child(source)
+	return &filer_pb.RemoteStorageLocation{
+		Name:   remoteMountLocation.Name,
+		Bucket: remoteMountLocation.Bucket,
+		Path:   string(dest),
+	}
+}
+
+func shouldSendToRemote(entry *filer_pb.Entry) bool {
+	if entry.RemoteEntry == nil {
+		return true
+	}
+	if entry.RemoteEntry.LastLocalSyncTsNs/1e9 < entry.Attributes.Mtime {
+		return true
+	}
+	return false
+}
+
+func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
+	entry.RemoteEntry = remoteEntry
+	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		_, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+			Directory: dir,
+			Entry:     entry,
+		})
+		return err
+	})
+}

weed/command/filer_replication.go

@@ -7,12 +7,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/replication"
 	"github.com/chrislusf/seaweedfs/weed/replication/sink"
-	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/azuresink"
-	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/b2sink"
-	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/filersink"
-	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink"
-	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink"
-	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink"
 	"github.com/chrislusf/seaweedfs/weed/replication/sub"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )

weed/command/filer_sync.go

@@ -15,7 +15,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/chrislusf/seaweedfs/weed/util/grace"
 	"google.golang.org/grpc"
-	"io"
 	"strings"
 	"time"
 )
@@ -71,8 +70,8 @@ func init() {
 
 var cmdFilerSynchronize = &Command{
 	UsageLine: "filer.sync -a=<oneFilerHost>:<oneFilerPort> -b=<otherFilerHost>:<otherFilerPort>",
-	Short:     "resumeable continuous synchronization between two active-active or active-passive SeaweedFS clusters",
-	Long: `resumeable continuous synchronization for file changes between two active-active or active-passive filers
+	Short:     "resumable continuous synchronization between two active-active or active-passive SeaweedFS clusters",
+	Long: `resumable continuous synchronization for file changes between two active-active or active-passive filers
 
 	filer.sync listens on filer notifications. If any file is updated, it will fetch the updated content,
 	and write to the other destination. Different from filer.replicate:
@@ -89,6 +88,7 @@ var cmdFilerSynchronize = &Command{
 
 func runFilerSynchronize(cmd *Command, args []string) bool {
 
+	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	grace.SetupProfiling(*syncCpuProfile, *syncMemProfile)
@@ -165,50 +165,14 @@ func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler, so
 		return persistEventFn(resp)
 	}
 
-	return pb.WithFilerClient(sourceFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-
-		ctx, cancel := context.WithCancel(context.Background())
-		defer cancel()
-
-		stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
-			ClientName: "syncTo_" + targetFiler,
-			PathPrefix: sourcePath,
-			SinceNs:    sourceFilerOffsetTsNs,
-			Signature:  targetFilerSignature,
-		})
-		if err != nil {
-			return fmt.Errorf("listen: %v", err)
-		}
-
-		var counter int64
-		var lastWriteTime time.Time
-		for {
-			resp, listenErr := stream.Recv()
-			if listenErr == io.EOF {
-				return nil
-			}
-			if listenErr != nil {
-				return listenErr
-			}
-
-			if err := processEventFn(resp); err != nil {
-				return err
-			}
-
-			counter++
-			if lastWriteTime.Add(3 * time.Second).Before(time.Now()) {
-				glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, resp.TsNs), float64(counter)/float64(3))
-				counter = 0
-				lastWriteTime = time.Now()
-				if err := setOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature, resp.TsNs); err != nil {
-					return err
-				}
-			}
-
-		}
-
+	processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error {
+		glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, lastTsNs), float64(counter)/float64(3))
+		return setOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature, lastTsNs)
 	})
 
+	return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+targetFiler,
+		sourcePath, sourceFilerOffsetTsNs, targetFilerSignature, processEventFnWithOffset, false)
+
 }
 
 const (

weed/command/fuse.go

@@ -2,10 +2,10 @@ package command
 
 import (
 	"fmt"
-	"strings"
-	"strconv"
-	"time"
 	"os"
+	"strconv"
+	"strings"
+	"time"
 )
 
 func init() {
@@ -13,7 +13,7 @@ func init() {
 }
 
 type parameter struct {
-	name string
+	name  string
 	value string
 }
 
@@ -23,6 +23,7 @@ func runFuse(cmd *Command, args []string) bool {
 	option := strings.Builder{}
 	options := []parameter{}
 	masterProcess := true
+	fusermountPath := ""
 
 	// first parameter
 	i := 0
@@ -41,7 +42,7 @@ func runFuse(cmd *Command, args []string) bool {
 				option.Reset()
 			}
 
-		// dash separator read option until next space
+			// dash separator read option until next space
 		} else if rawArgs[i] == '-' {
 			for i++; i < rawArgsLen && rawArgs[i] != ' '; i++ {
 				option.WriteByte(rawArgs[i])
@@ -49,25 +50,25 @@ func runFuse(cmd *Command, args []string) bool {
 			options = append(options, parameter{option.String(), "true"})
 			option.Reset()
 
-		// equal separator start option with pending value
+			// equal separator start option with pending value
 		} else if rawArgs[i] == '=' {
 			name := option.String()
 			option.Reset()
 
-			for i++; i < rawArgsLen && rawArgs[i] != ','; i++ {
+			for i++; i < rawArgsLen && rawArgs[i] != ',' && rawArgs[i] != ' '; i++ {
 				// double quote separator read option until next double quote
 				if rawArgs[i] == '"' {
 					for i++; i < rawArgsLen && rawArgs[i] != '"'; i++ {
 						option.WriteByte(rawArgs[i])
 					}
 
-				// single quote separator read option until next single quote
+					// single quote separator read option until next single quote
 				} else if rawArgs[i] == '\'' {
 					for i++; i < rawArgsLen && rawArgs[i] != '\''; i++ {
 						option.WriteByte(rawArgs[i])
 					}
 
-				// add chars before comma
+					// add chars before comma
 				} else if rawArgs[i] != ' ' {
 					option.WriteByte(rawArgs[i])
 				}
@@ -76,12 +77,12 @@ func runFuse(cmd *Command, args []string) bool {
 			options = append(options, parameter{name, option.String()})
 			option.Reset()
 
-		// comma separator just read current option
+			// comma separator just read current option
 		} else if rawArgs[i] == ',' {
 			options = append(options, parameter{option.String(), "true"})
 			option.Reset()
 
-		// what is not a separator fill option buffer
+			// what is not a separator fill option buffer
 		} else {
 			option.WriteByte(rawArgs[i])
 		}
@@ -98,7 +99,7 @@ func runFuse(cmd *Command, args []string) bool {
 	for i := 0; i < len(options); i++ {
 		parameter := options[i]
 
-		switch parameter.name  {
+		switch parameter.name {
 		case "child":
 			masterProcess = false
 		case "arg0":
@@ -108,7 +109,7 @@ func runFuse(cmd *Command, args []string) bool {
 		case "filer.path":
 			mountOptions.filerMountRootPath = &parameter.value
 		case "dirAutoCreate":
-			if parsed, err := strconv.ParseBool(parameter.value); err != nil {
+			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
 				mountOptions.dirAutoCreate = &parsed
 			} else {
 				panic(fmt.Errorf("dirAutoCreate: %s", err))
@@ -120,14 +121,14 @@ func runFuse(cmd *Command, args []string) bool {
 		case "disk":
 			mountOptions.diskType = &parameter.value
 		case "ttl":
-			if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err != nil {
+			if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err == nil {
 				intValue := int(parsed)
 				mountOptions.ttlSec = &intValue
 			} else {
 				panic(fmt.Errorf("ttl: %s", err))
 			}
 		case "chunkSizeLimitMB":
-			if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err != nil {
+			if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err == nil {
 				intValue := int(parsed)
 				mountOptions.chunkSizeLimitMB = &intValue
 			} else {
@@ -135,7 +136,7 @@ func runFuse(cmd *Command, args []string) bool {
 			}
 		case "concurrentWriters":
 			i++
-			if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err != nil {
+			if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err == nil {
 				intValue := int(parsed)
 				mountOptions.concurrentWriters = &intValue
 			} else {
@@ -144,7 +145,7 @@ func runFuse(cmd *Command, args []string) bool {
 		case "cacheDir":
 			mountOptions.cacheDir = &parameter.value
 		case "cacheCapacityMB":
-			if parsed, err := strconv.ParseInt(parameter.value, 0, 64); err != nil {
+			if parsed, err := strconv.ParseInt(parameter.value, 0, 64); err == nil {
 				mountOptions.cacheSizeMB = &parsed
 			} else {
 				panic(fmt.Errorf("cacheCapacityMB: %s", err))
@@ -152,7 +153,7 @@ func runFuse(cmd *Command, args []string) bool {
 		case "dataCenter":
 			mountOptions.dataCenter = &parameter.value
 		case "allowOthers":
-			if parsed, err := strconv.ParseBool(parameter.value); err != nil {
+			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
 				mountOptions.allowOthers = &parsed
 			} else {
 				panic(fmt.Errorf("allowOthers: %s", err))
@@ -160,7 +161,7 @@ func runFuse(cmd *Command, args []string) bool {
 		case "umask":
 			mountOptions.umaskString = &parameter.value
 		case "nonempty":
-			if parsed, err := strconv.ParseBool(parameter.value); err != nil {
+			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
 				mountOptions.nonempty = &parsed
 			} else {
 				panic(fmt.Errorf("nonempty: %s", err))
@@ -172,7 +173,7 @@ func runFuse(cmd *Command, args []string) bool {
 		case "map.gid":
 			mountOptions.gidMap = &parameter.value
 		case "readOnly":
-			if parsed, err := strconv.ParseBool(parameter.value); err != nil {
+			if parsed, err := strconv.ParseBool(parameter.value); err == nil {
 				mountOptions.readOnly = &parsed
 			} else {
 				panic(fmt.Errorf("readOnly: %s", err))
@@ -182,22 +183,28 @@ func runFuse(cmd *Command, args []string) bool {
 		case "memprofile":
 			mountMemProfile = &parameter.value
 		case "readRetryTime":
-			if parsed, err := time.ParseDuration(parameter.value); err != nil {
+			if parsed, err := time.ParseDuration(parameter.value); err == nil {
 				mountReadRetryTime = &parsed
 			} else {
 				panic(fmt.Errorf("readRetryTime: %s", err))
 			}
+		case "fusermount.path":
+			fusermountPath = parameter.value
 		}
 	}
 
 	// the master start the child, release it then finish himself
 	if masterProcess {
-		arg0 := os.Args[0]
+		arg0, err := os.Executable()
+		if err != nil {
+			panic(err)
+		}
+
 		argv := append(os.Args, "-o", "child")
 
-		attr := os.ProcAttr{}		
-		attr.Env = os.Environ() 
-		
+		attr := os.ProcAttr{}
+		attr.Env = os.Environ()
+
 		child, err := os.StartProcess(arg0, argv, &attr)
 
 		if err != nil {
@@ -213,13 +220,23 @@ func runFuse(cmd *Command, args []string) bool {
 		return true
 	}
 
+	if fusermountPath != "" {
+		if err := os.Setenv("PATH", fusermountPath); err != nil {
+			panic(fmt.Errorf("setenv: %s", err))
+		}
+	} else if os.Getenv("PATH") == "" {
+		if err := os.Setenv("PATH", "/bin:/sbin:/usr/bin:/usr/sbin"); err != nil {
+			panic(fmt.Errorf("setenv: %s", err))
+		}
+	}
+
 	// just call "weed mount" command
 	return runMount(cmdMount, []string{})
 }
 
 var cmdFuse = &Command{
 	UsageLine: "fuse /mnt/mount/point -o \"filer=localhost:8888,filer.path=/\"",
-	Short: "Allow use weed with linux's mount command",
+	Short:     "Allow use weed with linux's mount command",
 	Long: `Allow use weed with linux's mount command
 
   You can use -t weed on mount command:

weed/command/gateway.go

@@ -1,93 +0,0 @@
-package command
-
-import (
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/server"
-	"github.com/chrislusf/seaweedfs/weed/util"
-)
-
-var (
-	gatewayOptions GatewayOptions
-)
-
-type GatewayOptions struct {
-	masters *string
-	filers  *string
-	bindIp  *string
-	port    *int
-	maxMB   *int
-}
-
-func init() {
-	cmdGateway.Run = runGateway // break init cycle
-	gatewayOptions.masters = cmdGateway.Flag.String("master", "localhost:9333", "comma-separated master servers")
-	gatewayOptions.filers = cmdGateway.Flag.String("filer", "localhost:8888", "comma-separated filer servers")
-	gatewayOptions.bindIp = cmdGateway.Flag.String("ip.bind", "localhost", "ip address to bind to")
-	gatewayOptions.port = cmdGateway.Flag.Int("port", 5647, "gateway http listen port")
-	gatewayOptions.maxMB = cmdGateway.Flag.Int("maxMB", 4, "split files larger than the limit")
-}
-
-var cmdGateway = &Command{
-	UsageLine: "gateway -port=8888 -master=<ip:port>[,<ip:port>]* -filer=<ip:port>[,<ip:port>]*",
-	Short:     "start a gateway server that points to a list of master servers or a list of filers",
-	Long: `start a gateway server which accepts REST operation to write any blobs, files, or topic messages.
-
-	POST /blobs/
-		upload the blob and return a chunk id
-	DELETE /blobs/<chunk_id>
-		delete a chunk id
-
-	/*
-	POST /files/path/to/a/file
-		save /path/to/a/file on filer 
-	DELETE /files/path/to/a/file
-		delete /path/to/a/file on filer 
-
-	POST /topics/topicName
-		save on filer to /topics/topicName/<ds>/ts.json
-	*/
-`,
-}
-
-func runGateway(cmd *Command, args []string) bool {
-
-	util.LoadConfiguration("security", false)
-
-	gatewayOptions.startGateway()
-
-	return true
-}
-
-func (gw *GatewayOptions) startGateway() {
-
-	defaultMux := http.NewServeMux()
-
-	_, gws_err := weed_server.NewGatewayServer(defaultMux, &weed_server.GatewayOption{
-		Masters: strings.Split(*gw.masters, ","),
-		Filers:  strings.Split(*gw.filers, ","),
-		MaxMB:   *gw.maxMB,
-	})
-	if gws_err != nil {
-		glog.Fatalf("Gateway startup error: %v", gws_err)
-	}
-
-	glog.V(0).Infof("Start Seaweed Gateway %s at %s:%d", util.Version(), *gw.bindIp, *gw.port)
-	gatewayListener, e := util.NewListener(
-		*gw.bindIp+":"+strconv.Itoa(*gw.port),
-		time.Duration(10)*time.Second,
-	)
-	if e != nil {
-		glog.Fatalf("Filer listener error: %v", e)
-	}
-
-	httpS := &http.Server{Handler: defaultMux}
-	if err := httpS.Serve(gatewayListener); err != nil {
-		glog.Fatalf("Gateway Fail to serve: %v", e)
-	}
-
-}

weed/command/iam.go

@@ -49,6 +49,7 @@ func (iamopt *IamOptions) startIamServer() bool {
 		return false
 	}
 
+	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 	for {
 		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

weed/command/imports.go

@@ -0,0 +1,30 @@
+package command
+
+import (
+	_ "net/http/pprof"
+
+	_ "github.com/chrislusf/seaweedfs/weed/remote_storage/s3"
+
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/azuresink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/b2sink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/filersink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink"
+
+	_ "github.com/chrislusf/seaweedfs/weed/filer/cassandra"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/etcd"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/hbase"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb2"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb3"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/mongodb"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/mysql"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/mysql2"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/postgres"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/postgres2"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/redis"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/redis2"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/sqlite"
+)

weed/command/master.go

@@ -54,7 +54,7 @@ func init() {
 	m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
 	m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
 	// m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
-	m.defaultReplication = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
+	m.defaultReplication = cmdMaster.Flag.String("defaultReplication", "", "Default replication type if not specified.")
 	m.garbageThreshold = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	m.whiteList = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
 	m.disableHttp = cmdMaster.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
@@ -198,7 +198,7 @@ func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOp
 		Host:              *m.ip,
 		Port:              *m.port,
 		MetaFolder:        *m.metaFolder,
-		VolumeSizeLimitMB: *m.volumeSizeLimitMB,
+		VolumeSizeLimitMB: uint32(*m.volumeSizeLimitMB),
 		VolumePreallocate: *m.volumePreallocate,
 		// PulseSeconds:            *m.pulseSeconds,
 		DefaultReplicaPlacement: *m.defaultReplication,

weed/command/master_follower.go

@@ -0,0 +1,143 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/gorilla/mux"
+	"google.golang.org/grpc/reflection"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var (
+	mf MasterOptions
+)
+
+func init() {
+	cmdMasterFollower.Run = runMasterFollower // break init cycle
+	mf.port = cmdMasterFollower.Flag.Int("port", 9334, "http listen port")
+	mf.ipBind = cmdMasterFollower.Flag.String("ip.bind", "", "ip address to bind to")
+	mf.peers = cmdMasterFollower.Flag.String("masters", "localhost:9333", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
+
+	mf.ip = aws.String(util.DetectedHostAddress())
+	mf.metaFolder = aws.String("")
+	mf.volumeSizeLimitMB = nil
+	mf.volumePreallocate = nil
+	mf.defaultReplication = nil
+	mf.garbageThreshold = aws.Float64(0.1)
+	mf.whiteList = nil
+	mf.disableHttp = aws.Bool(false)
+	mf.metricsAddress = aws.String("")
+	mf.metricsIntervalSec = aws.Int(0)
+	mf.raftResumeState = aws.Bool(false)
+}
+
+var cmdMasterFollower = &Command{
+	UsageLine: "master.follower -port=9333 -masters=<master1Host>:<master1Port>",
+	Short:     "start a master follower",
+	Long: `start a master follower to provide volume=>location mapping service
+
+	The master follower does not participate in master election. 
+	It just follow the existing masters, and listen for any volume location changes.
+
+	In most cases, the master follower is not needed. In big data centers with thousands of volume
+	servers. In theory, the master may have trouble to keep up with the write requests and read requests.
+
+	The master follower can relieve the master from from read requests, which only needs to 
+	lookup a fileId or volumeId.
+
+	The master follower currently can handle fileId lookup requests:
+		/dir/lookup?volumeId=4
+		/dir/lookup?fileId=4,49c50924569199
+	And gRPC API
+		rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {}
+
+	This master follower is stateless and can run from any place.
+
+  `,
+}
+
+func runMasterFollower(cmd *Command, args []string) bool {
+
+	util.LoadConfiguration("security", false)
+	util.LoadConfiguration("master", false)
+
+	startMasterFollower(mf)
+
+	return true
+}
+
+func startMasterFollower(masterOptions MasterOptions) {
+
+	// collect settings from main masters
+	masters := strings.Split(*mf.peers, ",")
+	masterGrpcAddresses, err := pb.ParseServersToGrpcAddresses(masters)
+	if err != nil {
+		glog.V(0).Infof("ParseFilerGrpcAddress: %v", err)
+		return
+	}
+
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.master")
+	for i := 0; i < 10; i++ {
+		err = pb.WithOneOfGrpcMasterClients(masterGrpcAddresses, grpcDialOption, func(client master_pb.SeaweedClient) error {
+			resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
+			if err != nil {
+				return fmt.Errorf("get master grpc address %v configuration: %v", masterGrpcAddresses, err)
+			}
+			masterOptions.defaultReplication = &resp.DefaultReplication
+			masterOptions.volumeSizeLimitMB = aws.Uint(uint(resp.VolumeSizeLimitMB))
+			masterOptions.volumePreallocate = &resp.VolumePreallocate
+			return nil
+		})
+		if err != nil {
+			glog.V(0).Infof("failed to talk to filer %v: %v", masterGrpcAddresses, err)
+			glog.V(0).Infof("wait for %d seconds ...", i+1)
+			time.Sleep(time.Duration(i+1) * time.Second)
+		}
+	}
+	if err != nil {
+		glog.Errorf("failed to talk to filer %v: %v", masterGrpcAddresses, err)
+		return
+	}
+
+	option := masterOptions.toMasterOption(nil)
+	option.IsFollower = true
+
+	r := mux.NewRouter()
+	ms := weed_server.NewMasterServer(r, option, masters)
+	listeningAddress := *masterOptions.ipBind + ":" + strconv.Itoa(*masterOptions.port)
+	glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
+	masterListener, e := util.NewListener(listeningAddress, 0)
+	if e != nil {
+		glog.Fatalf("Master startup error: %v", e)
+	}
+
+	// starting grpc server
+	grpcPort := *masterOptions.port + 10000
+	grpcL, err := util.NewListener(*masterOptions.ipBind+":"+strconv.Itoa(grpcPort), 0)
+	if err != nil {
+		glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
+	}
+	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
+	master_pb.RegisterSeaweedServer(grpcS, ms)
+	reflection.Register(grpcS)
+	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOptions.ip, grpcPort)
+	go grpcS.Serve(grpcL)
+
+	go ms.MasterClient.KeepConnectedToMaster()
+
+	// start http server
+	httpS := &http.Server{Handler: r}
+	go httpS.Serve(masterListener)
+
+	select {}
+}

weed/command/scaffold.go

@@ -1,6 +1,8 @@
 package command
 
 import (
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/command/scaffold"
 	"io/ioutil"
 	"path/filepath"
 )
@@ -35,17 +37,17 @@ func runScaffold(cmd *Command, args []string) bool {
 	content := ""
 	switch *config {
 	case "filer":
-		content = FILER_TOML_EXAMPLE
+		content = scaffold.Filer
 	case "notification":
-		content = NOTIFICATION_TOML_EXAMPLE
+		content = scaffold.Notification
 	case "replication":
-		content = REPLICATION_TOML_EXAMPLE
+		content = scaffold.Replication
 	case "security":
-		content = SECURITY_TOML_EXAMPLE
+		content = scaffold.Security
 	case "master":
-		content = MASTER_TOML_EXAMPLE
+		content = scaffold.Master
 	case "shell":
-		content = SHELL_TOML_EXAMPLE
+		content = scaffold.Shell
 	}
 	if content == "" {
 		println("need a valid -config option")
@@ -55,519 +57,7 @@ func runScaffold(cmd *Command, args []string) bool {
 	if *outputPath != "" {
 		ioutil.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
 	} else {
-		println(content)
+		fmt.Println(content)
 	}
 	return true
 }
-
-const (
-	FILER_TOML_EXAMPLE = `
-# A sample TOML config file for SeaweedFS filer store
-# Used with "weed filer" or "weed server -filer"
-# Put this file to one of the location, with descending priority
-#    ./filer.toml
-#    $HOME/.seaweedfs/filer.toml
-#    /etc/seaweedfs/filer.toml
-
-####################################################
-# Customizable filer server options
-####################################################
-[filer.options]
-# with http DELETE, by default the filer would check whether a folder is empty.
-# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
-recursive_delete = false
-# directories under this folder will be automatically creating a separate bucket
-buckets_folder = "/buckets"
-
-####################################################
-# The following are filer store options
-####################################################
-
-[leveldb2]
-# local on disk, mostly for simple single-machine setup, fairly scalable
-# faster than previous leveldb, recommended.
-enabled = true
-dir = "./filerldb2"					# directory to store level db files
-
-[leveldb3]
-# similar to leveldb2.
-# each bucket has its own meta store.
-enabled = false
-dir = "./filerldb3"					# directory to store level db files
-
-[rocksdb]
-# local on disk, similar to leveldb
-# since it is using a C wrapper, you need to install rocksdb and build it by yourself
-enabled = false
-dir = "./filerrdb"					# directory to store rocksdb files
-
-[sqlite]
-# local on disk, similar to leveldb
-enabled = false
-dbFile = "./filer.db"				# sqlite db file
-
-[mysql]  # or memsql, tidb
-# CREATE TABLE IF NOT EXISTS filemeta (
-#   dirhash     BIGINT               COMMENT 'first 64 bits of MD5 hash value of directory field',
-#   name        VARCHAR(1000) BINARY COMMENT 'directory or file name',
-#   directory   TEXT                 COMMENT 'full path to parent directory',
-#   meta        LONGBLOB,
-#   PRIMARY KEY (dirhash, name)
-# ) DEFAULT CHARSET=utf8;
-
-enabled = false
-hostname = "localhost"
-port = 3306
-username = "root"
-password = ""
-database = ""              # create or use an existing database
-connection_max_idle = 2
-connection_max_open = 100
-connection_max_lifetime_seconds = 0
-interpolateParams = false
-# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
-enableUpsert = true
-upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
-
-[mysql2]  # or memsql, tidb
-enabled = false
-createTable = """
-  CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
-    dirhash BIGINT,
-    name VARCHAR(1000) BINARY,
-    directory TEXT,
-    meta LONGBLOB,
-    PRIMARY KEY (dirhash, name)
-  ) DEFAULT CHARSET=utf8;
-"""
-hostname = "localhost"
-port = 3306
-username = "root"
-password = ""
-database = ""              # create or use an existing database
-connection_max_idle = 2
-connection_max_open = 100
-connection_max_lifetime_seconds = 0
-interpolateParams = false
-# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
-enableUpsert = true
-upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
-
-[postgres] # or cockroachdb, YugabyteDB
-# CREATE TABLE IF NOT EXISTS filemeta (
-#   dirhash     BIGINT,
-#   name        VARCHAR(65535),
-#   directory   VARCHAR(65535),
-#   meta        bytea,
-#   PRIMARY KEY (dirhash, name)
-# );
-enabled = false
-hostname = "localhost"
-port = 5432
-username = "postgres"
-password = ""
-database = "postgres"          # create or use an existing database
-schema = ""
-sslmode = "disable"
-connection_max_idle = 100
-connection_max_open = 100
-connection_max_lifetime_seconds = 0
-# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
-enableUpsert = true
-upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
-
-[postgres2]
-enabled = false
-createTable = """
-  CREATE TABLE IF NOT EXISTS "%s" (
-    dirhash   BIGINT, 
-    name      VARCHAR(65535), 
-    directory VARCHAR(65535), 
-    meta      bytea, 
-    PRIMARY KEY (dirhash, name)
-  );
-"""
-hostname = "localhost"
-port = 5432
-username = "postgres"
-password = ""
-database = "postgres"          # create or use an existing database
-schema = ""
-sslmode = "disable"
-connection_max_idle = 100
-connection_max_open = 100
-connection_max_lifetime_seconds = 0
-# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
-enableUpsert = true
-upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
-
-[cassandra]
-# CREATE TABLE filemeta (
-#    directory varchar,
-#    name varchar,
-#    meta blob,
-#    PRIMARY KEY (directory, name)
-# ) WITH CLUSTERING ORDER BY (name ASC);
-enabled = false
-keyspace="seaweedfs"
-hosts=[
-	"localhost:9042",
-]
-username=""
-password=""
-# This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
-superLargeDirectories = []
-
-[hbase]
-enabled = false
-zkquorum = ""
-table = "seaweedfs"
-
-[redis2]
-enabled = false
-address  = "localhost:6379"
-password = ""
-database = 0
-# This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
-superLargeDirectories = []
-
-[redis_cluster2]
-enabled = false
-addresses = [
-    "localhost:30001",
-    "localhost:30002",
-    "localhost:30003",
-    "localhost:30004",
-    "localhost:30005",
-    "localhost:30006",
-]
-password = ""
-# allows reads from slave servers or the master, but all writes still go to the master
-readOnly = false
-# automatically use the closest Redis server for reads
-routeByLatency = false
-# This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
-superLargeDirectories = []
-
-[etcd]
-enabled = false
-servers = "localhost:2379"
-timeout = "3s"
-
-[mongodb]
-enabled = false
-uri = "mongodb://localhost:27017"
-option_pool_size = 0
-database = "seaweedfs"
-
-[elastic7]
-enabled = false
-servers = [
-    "http://localhost1:9200",
-    "http://localhost2:9200",
-    "http://localhost3:9200",
-]
-username = ""
-password = ""
-sniff_enabled = false
-healthcheck_enabled = false
-# increase the value is recommend, be sure the value in Elastic is greater or equal here
-index.max_result_window = 10000
-
-
-
-##########################
-##########################
-# To add path-specific filer store:
-#
-# 1. Add a name following the store type separated by a dot ".". E.g., cassandra.tmp
-# 2. Add a location configuraiton. E.g., location = "/tmp/"
-# 3. Copy and customize all other configurations. 
-#     Make sure they are not the same if using the same store type!
-# 4. Set enabled to true
-#
-# The following is just using redis as an example
-##########################
-[redis2.tmp]
-enabled = false
-location = "/tmp/"
-address  = "localhost:6379"
-password = ""
-database = 1
-
-`
-
-	NOTIFICATION_TOML_EXAMPLE = `
-# A sample TOML config file for SeaweedFS filer store
-# Used by both "weed filer" or "weed server -filer" and "weed filer.replicate"
-# Put this file to one of the location, with descending priority
-#    ./notification.toml
-#    $HOME/.seaweedfs/notification.toml
-#    /etc/seaweedfs/notification.toml
-
-####################################################
-# notification
-# send and receive filer updates for each file to an external message queue
-####################################################
-[notification.log]
-# this is only for debugging perpose and does not work with "weed filer.replicate"
-enabled = false
-
-
-[notification.kafka]
-enabled = false
-hosts = [
-  "localhost:9092"
-]
-topic = "seaweedfs_filer"
-offsetFile = "./last.offset"
-offsetSaveIntervalSeconds = 10
-
-
-[notification.aws_sqs]
-# experimental, let me know if it works
-enabled = false
-aws_access_key_id     = ""        # if empty, loads from the shared credentials file (~/.aws/credentials).
-aws_secret_access_key = ""        # if empty, loads from the shared credentials file (~/.aws/credentials).
-region = "us-east-2"
-sqs_queue_name = "my_filer_queue" # an existing queue name
-
-
-[notification.google_pub_sub]
-# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
-enabled = false
-google_application_credentials = "/path/to/x.json" # path to json credential file
-project_id = ""                       # an existing project id
-topic = "seaweedfs_filer_topic"       # a topic, auto created if does not exists
-
-[notification.gocdk_pub_sub]
-# The Go Cloud Development Kit (https://gocloud.dev).
-# PubSub API (https://godoc.org/gocloud.dev/pubsub).
-# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
-enabled = false
-# This URL will Dial the RabbitMQ server at the URL in the environment
-# variable RABBIT_SERVER_URL and open the exchange "myexchange".
-# The exchange must have already been created by some other means, like
-# the RabbitMQ management plugin. Сreate myexchange of type fanout and myqueue then
-# create binding myexchange => myqueue
-topic_url = "rabbit://myexchange"
-sub_url = "rabbit://myqueue"
-`
-
-	REPLICATION_TOML_EXAMPLE = `
-# A sample TOML config file for replicating SeaweedFS filer
-# Used with "weed filer.backup"
-# Using with "weed filer.replicate" is deprecated.
-# Put this file to one of the location, with descending priority
-#    ./replication.toml
-#    $HOME/.seaweedfs/replication.toml
-#    /etc/seaweedfs/replication.toml
-
-[source.filer]  # deprecated. Only useful with "weed filer.replicate"
-enabled = true
-grpcAddress = "localhost:18888"
-# all files under this directory tree are replicated.
-# this is not a directory on your hard drive, but on your filer.
-# i.e., all files with this "prefix" are sent to notification message queue.
-directory = "/buckets"
-
-[sink.local]
-enabled = false
-directory = "/data"
-# all replicated files are under modified time as yyyy-mm-dd directories
-# so each date directory contains all new and updated files.
-is_incremental = false
-
-[sink.filer]
-enabled = false
-grpcAddress = "localhost:18888"
-# all replicated files are under this directory tree
-# this is not a directory on your hard drive, but on your filer.
-# i.e., all received files will be "prefixed" to this directory.
-directory = "/backup"
-replication = ""
-collection = ""
-ttlSec = 0
-is_incremental = false
-
-[sink.s3]
-# read credentials doc at https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sessions.html
-# default loads credentials from the shared credentials file (~/.aws/credentials).
-enabled = false
-aws_access_key_id     = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
-aws_secret_access_key = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
-region = "us-east-2"
-bucket = "your_bucket_name"    # an existing bucket
-directory = "/"                # destination directory
-endpoint = ""
-is_incremental = false
-
-[sink.google_cloud_storage]
-# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
-enabled = false
-google_application_credentials = "/path/to/x.json" # path to json credential file
-bucket = "your_bucket_seaweedfs"    # an existing bucket
-directory = "/"                     # destination directory
-is_incremental = false
-
-[sink.azure]
-# experimental, let me know if it works
-enabled = false
-account_name = ""
-account_key  = ""
-container = "mycontainer"      # an existing container
-directory = "/"                # destination directory
-is_incremental = false
-
-[sink.backblaze]
-enabled = false
-b2_account_id = ""
-b2_master_application_key  = ""
-bucket = "mybucket"            # an existing bucket
-directory = "/"                # destination directory
-is_incremental = false
-
-`
-
-	SECURITY_TOML_EXAMPLE = `
-# Put this file to one of the location, with descending priority
-#    ./security.toml
-#    $HOME/.seaweedfs/security.toml
-#    /etc/seaweedfs/security.toml
-# this file is read by master, volume server, and filer
-
-# the jwt signing key is read by master and volume server.
-# a jwt defaults to expire after 10 seconds.
-[jwt.signing]
-key = ""
-expires_after_seconds = 10           # seconds
-
-# jwt for read is only supported with master+volume setup. Filer does not support this mode.
-[jwt.signing.read]
-key = ""
-expires_after_seconds = 10           # seconds
-
-# all grpc tls authentications are mutual
-# the values for the following ca, cert, and key are paths to the PERM files.
-# the host name is not checked, so the PERM files can be shared.
-[grpc]
-ca = ""
-# Set wildcard domain for enable TLS authentication by common names
-allowed_wildcard_domain = "" # .mycompany.com
-
-[grpc.volume]
-cert = ""
-key  = ""
-allowed_commonNames = ""	# comma-separated SSL certificate common names
-
-[grpc.master]
-cert = ""
-key  = ""
-allowed_commonNames = ""	# comma-separated SSL certificate common names
-
-[grpc.filer]
-cert = ""
-key  = ""
-allowed_commonNames = ""	# comma-separated SSL certificate common names
-
-[grpc.msg_broker]
-cert = ""
-key  = ""
-allowed_commonNames = ""	# comma-separated SSL certificate common names
-
-# use this for any place needs a grpc client
-# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
-[grpc.client]
-cert = ""
-key  = ""
-
-# volume server https options
-# Note: work in progress!
-#     this does not work with other clients, e.g., "weed filer|mount" etc, yet.
-[https.client]
-enabled = true
-[https.volume]
-cert = ""
-key  = ""
-
-
-`
-
-	MASTER_TOML_EXAMPLE = `
-# Put this file to one of the location, with descending priority
-#    ./master.toml
-#    $HOME/.seaweedfs/master.toml
-#    /etc/seaweedfs/master.toml
-# this file is read by master
-
-[master.maintenance]
-# periodically run these scripts are the same as running them from 'weed shell'
-scripts = """
-  lock
-  ec.encode -fullPercent=95 -quietFor=1h
-  ec.rebuild -force
-  ec.balance -force
-  volume.balance -force
-  volume.fix.replication
-  unlock
-"""
-sleep_minutes = 17          # sleep minutes between each script execution
-
-[master.filer]
-default = "localhost:8888"    # used by maintenance scripts if the scripts needs to use fs related commands
-
-
-[master.sequencer]
-type = "raft"     # Choose [raft|etcd|snowflake] type for storing the file id sequence
-# when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
-# example : http://127.0.0.1:2379,http://127.0.0.1:2389
-sequencer_etcd_urls = "http://127.0.0.1:2379"
-
-
-# configurations for tiered cloud storage
-# old volumes are transparently moved to cloud for cost efficiency
-[storage.backend]
-	[storage.backend.s3.default]
-	enabled = false
-	aws_access_key_id     = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
-	aws_secret_access_key = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
-	region = "us-east-2"
-	bucket = "your_bucket_name"    # an existing bucket
-	endpoint = ""
-
-# create this number of logical volumes if no more writable volumes
-# count_x means how many copies of data.
-# e.g.:
-#   000 has only one copy, copy_1
-#   010 and 001 has two copies, copy_2
-#   011 has only 3 copies, copy_3
-[master.volume_growth]
-copy_1 = 7                # create 1 x 7 = 7 actual volumes
-copy_2 = 6                # create 2 x 6 = 12 actual volumes
-copy_3 = 3                # create 3 x 3 = 9 actual volumes
-copy_other = 1            # create n x 1 = n actual volumes
-
-# configuration flags for replication
-[master.replication]
-# any replication counts should be considered minimums. If you specify 010 and
-# have 3 different racks, that's still considered writable. Writes will still
-# try to replicate to all available volumes. You should only use this option
-# if you are doing your own replication or periodic sync of volumes.
-treat_replication_as_minimums = false
-
-`
-	SHELL_TOML_EXAMPLE = `
-
-[cluster]
-default = "c1"
-
-[cluster.c1]
-master = "localhost:9333"    # comma-separated master servers
-filer = "localhost:8888"     # filer host and port
-
-[cluster.c2]
-master = ""
-filer = ""
-
-`
-)

weed/command/scaffold/example.go

@@ -0,0 +1,21 @@
+package scaffold
+
+import _ "embed"
+
+//go:embed filer.toml
+var Filer string
+
+//go:embed notification.toml
+var Notification string
+
+//go:embed replication.toml
+var Replication string
+
+//go:embed security.toml
+var Security string
+
+//go:embed master.toml
+var Master string
+
+//go:embed shell.toml
+var Shell string

weed/command/scaffold/filer.toml

@@ -0,0 +1,232 @@
+# A sample TOML config file for SeaweedFS filer store
+# Used with "weed filer" or "weed server -filer"
+# Put this file to one of the location, with descending priority
+#    ./filer.toml
+#    $HOME/.seaweedfs/filer.toml
+#    /etc/seaweedfs/filer.toml
+
+####################################################
+# Customizable filer server options
+####################################################
+[filer.options]
+# with http DELETE, by default the filer would check whether a folder is empty.
+# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
+recursive_delete = false
+
+####################################################
+# The following are filer store options
+####################################################
+
+[leveldb2]
+# local on disk, mostly for simple single-machine setup, fairly scalable
+# faster than previous leveldb, recommended.
+enabled = true
+dir = "./filerldb2"                    # directory to store level db files
+
+[leveldb3]
+# similar to leveldb2.
+# each bucket has its own meta store.
+enabled = false
+dir = "./filerldb3"                    # directory to store level db files
+
+[rocksdb]
+# local on disk, similar to leveldb
+# since it is using a C wrapper, you need to install rocksdb and build it by yourself
+enabled = false
+dir = "./filerrdb"                    # directory to store rocksdb files
+
+[sqlite]
+# local on disk, similar to leveldb
+enabled = false
+dbFile = "./filer.db"                # sqlite db file
+
+[mysql]  # or memsql, tidb
+# CREATE TABLE IF NOT EXISTS filemeta (
+#   dirhash     BIGINT               COMMENT 'first 64 bits of MD5 hash value of directory field',
+#   name        VARCHAR(1000) BINARY COMMENT 'directory or file name',
+#   directory   TEXT                 COMMENT 'full path to parent directory',
+#   meta        LONGBLOB,
+#   PRIMARY KEY (dirhash, name)
+# ) DEFAULT CHARSET=utf8;
+
+enabled = false
+hostname = "localhost"
+port = 3306
+username = "root"
+password = ""
+database = ""              # create or use an existing database
+connection_max_idle = 2
+connection_max_open = 100
+connection_max_lifetime_seconds = 0
+interpolateParams = false
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
+
+[mysql2]  # or memsql, tidb
+enabled = false
+createTable = """
+  CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
+    dirhash BIGINT,
+    name VARCHAR(1000) BINARY,
+    directory TEXT,
+    meta LONGBLOB,
+    PRIMARY KEY (dirhash, name)
+  ) DEFAULT CHARSET=utf8;
+"""
+hostname = "localhost"
+port = 3306
+username = "root"
+password = ""
+database = ""              # create or use an existing database
+connection_max_idle = 2
+connection_max_open = 100
+connection_max_lifetime_seconds = 0
+interpolateParams = false
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
+
+[postgres] # or cockroachdb, YugabyteDB
+# CREATE TABLE IF NOT EXISTS filemeta (
+#   dirhash     BIGINT,
+#   name        VARCHAR(65535),
+#   directory   VARCHAR(65535),
+#   meta        bytea,
+#   PRIMARY KEY (dirhash, name)
+# );
+enabled = false
+hostname = "localhost"
+port = 5432
+username = "postgres"
+password = ""
+database = "postgres"          # create or use an existing database
+schema = ""
+sslmode = "disable"
+connection_max_idle = 100
+connection_max_open = 100
+connection_max_lifetime_seconds = 0
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
+
+[postgres2]
+enabled = false
+createTable = """
+  CREATE TABLE IF NOT EXISTS "%s" (
+    dirhash   BIGINT,
+    name      VARCHAR(65535),
+    directory VARCHAR(65535),
+    meta      bytea,
+    PRIMARY KEY (dirhash, name)
+  );
+"""
+hostname = "localhost"
+port = 5432
+username = "postgres"
+password = ""
+database = "postgres"          # create or use an existing database
+schema = ""
+sslmode = "disable"
+connection_max_idle = 100
+connection_max_open = 100
+connection_max_lifetime_seconds = 0
+# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
+enableUpsert = true
+upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
+
+[cassandra]
+# CREATE TABLE filemeta (
+#    directory varchar,
+#    name varchar,
+#    meta blob,
+#    PRIMARY KEY (directory, name)
+# ) WITH CLUSTERING ORDER BY (name ASC);
+enabled = false
+keyspace = "seaweedfs"
+hosts = [
+    "localhost:9042",
+]
+username = ""
+password = ""
+# This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
+superLargeDirectories = []
+# Name of the datacenter local to this filer, used as host selection fallback.
+localDC = ""
+
+[hbase]
+enabled = false
+zkquorum = ""
+table = "seaweedfs"
+
+[redis2]
+enabled = false
+address = "localhost:6379"
+password = ""
+database = 0
+# This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
+superLargeDirectories = []
+
+[redis_cluster2]
+enabled = false
+addresses = [
+    "localhost:30001",
+    "localhost:30002",
+    "localhost:30003",
+    "localhost:30004",
+    "localhost:30005",
+    "localhost:30006",
+]
+password = ""
+# allows reads from slave servers or the master, but all writes still go to the master
+readOnly = false
+# automatically use the closest Redis server for reads
+routeByLatency = false
+# This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
+superLargeDirectories = []
+
+[etcd]
+enabled = false
+servers = "localhost:2379"
+timeout = "3s"
+
+[mongodb]
+enabled = false
+uri = "mongodb://localhost:27017"
+option_pool_size = 0
+database = "seaweedfs"
+
+[elastic7]
+enabled = false
+servers = [
+    "http://localhost1:9200",
+    "http://localhost2:9200",
+    "http://localhost3:9200",
+]
+username = ""
+password = ""
+sniff_enabled = false
+healthcheck_enabled = false
+# increase the value is recommend, be sure the value in Elastic is greater or equal here
+index.max_result_window = 10000
+
+
+
+##########################
+##########################
+# To add path-specific filer store:
+#
+# 1. Add a name following the store type separated by a dot ".". E.g., cassandra.tmp
+# 2. Add a location configuraiton. E.g., location = "/tmp/"
+# 3. Copy and customize all other configurations.
+#     Make sure they are not the same if using the same store type!
+# 4. Set enabled to true
+#
+# The following is just using redis as an example
+##########################
+[redis2.tmp]
+enabled = false
+location = "/tmp/"
+address = "localhost:6379"
+password = ""
+database = 1

weed/command/scaffold/master.toml

@@ -0,0 +1,63 @@
+# Put this file to one of the location, with descending priority
+#    ./master.toml
+#    $HOME/.seaweedfs/master.toml
+#    /etc/seaweedfs/master.toml
+# this file is read by master
+
+[master.maintenance]
+# periodically run these scripts are the same as running them from 'weed shell'
+scripts = """
+  lock
+  ec.encode -fullPercent=95 -quietFor=1h
+  ec.rebuild -force
+  ec.balance -force
+  volume.deleteEmpty -quietFor=24h -force
+  volume.balance -force
+  volume.fix.replication
+  unlock
+"""
+sleep_minutes = 17          # sleep minutes between each script execution
+
+[master.filer]
+default = "localhost:8888"    # used by maintenance scripts if the scripts needs to use fs related commands
+
+
+[master.sequencer]
+type = "raft"     # Choose [raft|etcd|snowflake] type for storing the file id sequence
+# when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
+# example : http://127.0.0.1:2379,http://127.0.0.1:2389
+sequencer_etcd_urls = "http://127.0.0.1:2379"
+# when sequencer.type = snowflake, the snowflake id must be different from other masters
+sequencer_snowflake_id = 0     # any number between 1~1023
+
+
+# configurations for tiered cloud storage
+# old volumes are transparently moved to cloud for cost efficiency
+[storage.backend]
+[storage.backend.s3.default]
+enabled = false
+aws_access_key_id = ""         # if empty, loads from the shared credentials file (~/.aws/credentials).
+aws_secret_access_key = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
+region = "us-east-2"
+bucket = "your_bucket_name"    # an existing bucket
+endpoint = ""
+
+# create this number of logical volumes if no more writable volumes
+# count_x means how many copies of data.
+# e.g.:
+#   000 has only one copy, copy_1
+#   010 and 001 has two copies, copy_2
+#   011 has only 3 copies, copy_3
+[master.volume_growth]
+copy_1 = 7                # create 1 x 7 = 7 actual volumes
+copy_2 = 6                # create 2 x 6 = 12 actual volumes
+copy_3 = 3                # create 3 x 3 = 9 actual volumes
+copy_other = 1            # create n x 1 = n actual volumes
+
+# configuration flags for replication
+[master.replication]
+# any replication counts should be considered minimums. If you specify 010 and
+# have 3 different racks, that's still considered writable. Writes will still
+# try to replicate to all available volumes. You should only use this option
+# if you are doing your own replication or periodic sync of volumes.
+treat_replication_as_minimums = false

weed/command/scaffold/notification.toml

@@ -0,0 +1,54 @@
+# A sample TOML config file for SeaweedFS filer store
+# Used by both "weed filer" or "weed server -filer" and "weed filer.replicate"
+# Put this file to one of the location, with descending priority
+#    ./notification.toml
+#    $HOME/.seaweedfs/notification.toml
+#    /etc/seaweedfs/notification.toml
+
+####################################################
+# notification
+# send and receive filer updates for each file to an external message queue
+####################################################
+[notification.log]
+# this is only for debugging perpose and does not work with "weed filer.replicate"
+enabled = false
+
+
+[notification.kafka]
+enabled = false
+hosts = [
+    "localhost:9092"
+]
+topic = "seaweedfs_filer"
+offsetFile = "./last.offset"
+offsetSaveIntervalSeconds = 10
+
+
+[notification.aws_sqs]
+# experimental, let me know if it works
+enabled = false
+aws_access_key_id = ""        # if empty, loads from the shared credentials file (~/.aws/credentials).
+aws_secret_access_key = ""        # if empty, loads from the shared credentials file (~/.aws/credentials).
+region = "us-east-2"
+sqs_queue_name = "my_filer_queue" # an existing queue name
+
+
+[notification.google_pub_sub]
+# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
+enabled = false
+google_application_credentials = "/path/to/x.json" # path to json credential file
+project_id = ""                       # an existing project id
+topic = "seaweedfs_filer_topic"       # a topic, auto created if does not exists
+
+[notification.gocdk_pub_sub]
+# The Go Cloud Development Kit (https://gocloud.dev).
+# PubSub API (https://godoc.org/gocloud.dev/pubsub).
+# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
+enabled = false
+# This URL will Dial the RabbitMQ server at the URL in the environment
+# variable RABBIT_SERVER_URL and open the exchange "myexchange".
+# The exchange must have already been created by some other means, like
+# the RabbitMQ management plugin. Сreate myexchange of type fanout and myqueue then
+# create binding myexchange => myqueue
+topic_url = "rabbit://myexchange"
+sub_url = "rabbit://myqueue"

weed/command/scaffold/replication.toml

@@ -0,0 +1,71 @@
+# A sample TOML config file for replicating SeaweedFS filer
+# Used with "weed filer.backup"
+# Using with "weed filer.replicate" is deprecated.
+# Put this file to one of the location, with descending priority
+#    ./replication.toml
+#    $HOME/.seaweedfs/replication.toml
+#    /etc/seaweedfs/replication.toml
+
+[source.filer]  # deprecated. Only useful with "weed filer.replicate"
+enabled = true
+grpcAddress = "localhost:18888"
+# all files under this directory tree are replicated.
+# this is not a directory on your hard drive, but on your filer.
+# i.e., all files with this "prefix" are sent to notification message queue.
+directory = "/buckets"
+
+[sink.local]
+enabled = false
+directory = "/data"
+# all replicated files are under modified time as yyyy-mm-dd directories
+# so each date directory contains all new and updated files.
+is_incremental = false
+
+[sink.filer]
+enabled = false
+grpcAddress = "localhost:18888"
+# all replicated files are under this directory tree
+# this is not a directory on your hard drive, but on your filer.
+# i.e., all received files will be "prefixed" to this directory.
+directory = "/backup"
+replication = ""
+collection = ""
+ttlSec = 0
+is_incremental = false
+
+[sink.s3]
+# read credentials doc at https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sessions.html
+# default loads credentials from the shared credentials file (~/.aws/credentials).
+enabled = false
+aws_access_key_id = ""         # if empty, loads from the shared credentials file (~/.aws/credentials).
+aws_secret_access_key = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
+region = "us-east-2"
+bucket = "your_bucket_name"    # an existing bucket
+directory = "/"                # destination directory
+endpoint = ""
+is_incremental = false
+
+[sink.google_cloud_storage]
+# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
+enabled = false
+google_application_credentials = "/path/to/x.json" # path to json credential file
+bucket = "your_bucket_seaweedfs"    # an existing bucket
+directory = "/"                     # destination directory
+is_incremental = false
+
+[sink.azure]
+# experimental, let me know if it works
+enabled = false
+account_name = ""
+account_key = ""
+container = "mycontainer"      # an existing container
+directory = "/"                # destination directory
+is_incremental = false
+
+[sink.backblaze]
+enabled = false
+b2_account_id = ""
+b2_master_application_key = ""
+bucket = "mybucket"            # an existing bucket
+directory = "/"                # destination directory
+is_incremental = false

weed/command/scaffold/security.toml

@@ -0,0 +1,60 @@
+# Put this file to one of the location, with descending priority
+#    ./security.toml
+#    $HOME/.seaweedfs/security.toml
+#    /etc/seaweedfs/security.toml
+# this file is read by master, volume server, and filer
+
+# the jwt signing key is read by master and volume server.
+# a jwt defaults to expire after 10 seconds.
+[jwt.signing]
+key = ""
+expires_after_seconds = 10           # seconds
+
+# jwt for read is only supported with master+volume setup. Filer does not support this mode.
+[jwt.signing.read]
+key = ""
+expires_after_seconds = 10           # seconds
+
+# all grpc tls authentications are mutual
+# the values for the following ca, cert, and key are paths to the PERM files.
+# the host name is not checked, so the PERM files can be shared.
+[grpc]
+ca = ""
+# Set wildcard domain for enable TLS authentication by common names
+allowed_wildcard_domain = "" # .mycompany.com
+
+[grpc.volume]
+cert = ""
+key = ""
+allowed_commonNames = ""    # comma-separated SSL certificate common names
+
+[grpc.master]
+cert = ""
+key = ""
+allowed_commonNames = ""    # comma-separated SSL certificate common names
+
+[grpc.filer]
+cert = ""
+key = ""
+allowed_commonNames = ""    # comma-separated SSL certificate common names
+
+[grpc.msg_broker]
+cert = ""
+key = ""
+allowed_commonNames = ""    # comma-separated SSL certificate common names
+
+# use this for any place needs a grpc client
+# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
+[grpc.client]
+cert = ""
+key = ""
+
+# volume server https options
+# Note: work in progress!
+#     this does not work with other clients, e.g., "weed filer|mount" etc, yet.
+[https.client]
+enabled = true
+[https.volume]
+cert = ""
+key = ""
+

weed/command/scaffold/shell.toml

@@ -0,0 +1,10 @@
+[cluster]
+default = "c1"
+
+[cluster.c1]
+master = "localhost:9333"    # comma-separated master servers
+filer = "localhost:8888"     # filer host and port
+
+[cluster.c2]
+master = ""
+filer = ""

weed/command/server.go

@@ -3,6 +3,7 @@ package command
 import (
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/util/grace"
+	"net/http"
 	"os"
 	"strings"
 	"time"
@@ -16,6 +17,8 @@ import (
 type ServerOptions struct {
 	cpuprofile *string
 	memprofile *string
+	debug      *bool
+	debugPort  *int
 	v          VolumeServerOptions
 }
 
@@ -78,13 +81,15 @@ var (
 func init() {
 	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
 	serverOptions.memprofile = cmdServer.Flag.String("memprofile", "", "memory profile output file")
+	serverOptions.debug = cmdServer.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:6060/debug/pprof/goroutine?debug=2")
+	serverOptions.debugPort = cmdServer.Flag.Int("debug.port", 6060, "http port for debugging")
 
 	masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
 	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
 	masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
 	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
 	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
-	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "000", "Default replication type if not specified.")
+	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
 	masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
 	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
@@ -107,10 +112,11 @@ func init() {
 	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	serverOptions.v.diskType = cmdServer.Flag.String("volume.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
-	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
+	serverOptions.v.readMode = cmdServer.Flag.String("volume.readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|read in remote node|redirect volume location'.")
 	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
 	serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
 	serverOptions.v.concurrentUploadLimitMB = cmdServer.Flag.Int("volume.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
+	serverOptions.v.concurrentDownloadLimitMB = cmdServer.Flag.Int("volume.concurrentDownloadLimitMB", 64, "limit total concurrent download size")
 	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
 	serverOptions.v.preStopSeconds = cmdServer.Flag.Int("volume.preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
 	serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
@@ -139,6 +145,10 @@ func init() {
 
 func runServer(cmd *Command, args []string) bool {
 
+	if *serverOptions.debug {
+		go http.ListenAndServe(fmt.Sprintf(":%d", *serverOptions.debugPort), nil)
+	}
+
 	util.LoadConfiguration("security", false)
 	util.LoadConfiguration("master", false)
 

weed/command/shell.go

@@ -55,6 +55,7 @@ func runShell(command *Command, args []string) bool {
 
 	var err error
 	shellOptions.FilerHost, shellOptions.FilerPort, err = util.ParseHostPort(*shellInitialFiler)
+	shellOptions.FilerAddress = *shellInitialFiler
 	if err != nil {
 		fmt.Printf("failed to parse filer %s: %v\n", *shellInitialFiler, err)
 		return false

weed/command/upload.go

@@ -71,13 +71,13 @@ func runUpload(cmd *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
-	defaultCollection, err := readMasterConfiguration(grpcDialOption, *upload.master)
+	defaultReplication, err := readMasterConfiguration(grpcDialOption, *upload.master)
 	if err != nil {
 		fmt.Printf("upload: %v", err)
 		return false
 	}
 	if *upload.replication == "" {
-		*upload.replication = defaultCollection
+		*upload.replication = defaultReplication
 	}
 
 	if len(args) == 0 {
@@ -110,7 +110,7 @@ func runUpload(cmd *Command, args []string) bool {
 		})
 		if err != nil {
 			fmt.Println(err.Error())
-			return false;
+			return false
 		}
 	} else {
 		parts, e := operation.NewFileParts(args)

weed/command/volume.go

@@ -35,31 +35,32 @@ var (
 )
 
 type VolumeServerOptions struct {
-	port                    *int
-	publicPort              *int
-	folders                 []string
-	folderMaxLimits         []int
-	idxFolder               *string
-	ip                      *string
-	publicUrl               *string
-	bindIp                  *string
-	masters                 *string
-	idleConnectionTimeout   *int
-	dataCenter              *string
-	rack                    *string
-	whiteList               []string
-	indexType               *string
-	diskType                *string
-	fixJpgOrientation       *bool
-	readRedirect            *bool
-	cpuProfile              *string
-	memProfile              *string
-	compactionMBPerSecond   *int
-	fileSizeLimitMB         *int
-	concurrentUploadLimitMB *int
-	pprof                   *bool
-	preStopSeconds          *int
-	metricsHttpPort         *int
+	port                      *int
+	publicPort                *int
+	folders                   []string
+	folderMaxLimits           []int
+	idxFolder                 *string
+	ip                        *string
+	publicUrl                 *string
+	bindIp                    *string
+	masters                   *string
+	idleConnectionTimeout     *int
+	dataCenter                *string
+	rack                      *string
+	whiteList                 []string
+	indexType                 *string
+	diskType                  *string
+	fixJpgOrientation         *bool
+	readMode                  *string
+	cpuProfile                *string
+	memProfile                *string
+	compactionMBPerSecond     *int
+	fileSizeLimitMB           *int
+	concurrentUploadLimitMB   *int
+	concurrentDownloadLimitMB *int
+	pprof                     *bool
+	preStopSeconds            *int
+	metricsHttpPort           *int
 	// pulseSeconds          *int
 	enableTcp *bool
 }
@@ -80,12 +81,13 @@ func init() {
 	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	v.diskType = cmdVolume.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
-	v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
+	v.readMode = cmdVolume.Flag.String("readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|proxy to remote node|redirect volume location'.")
 	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
 	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
 	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
 	v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
-	v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size")
+	v.concurrentUploadLimitMB = cmdVolume.Flag.Int("concurrentUploadLimitMB", 256, "limit total concurrent upload size")
+	v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 256, "limit total concurrent download size")
 	v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
 	v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
@@ -228,10 +230,11 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		volumeNeedleMapKind,
 		strings.Split(masters, ","), 5, *v.dataCenter, *v.rack,
 		v.whiteList,
-		*v.fixJpgOrientation, *v.readRedirect,
+		*v.fixJpgOrientation, *v.readMode,
 		*v.compactionMBPerSecond,
 		*v.fileSizeLimitMB,
 		int64(*v.concurrentUploadLimitMB)*1024*1024,
+		int64(*v.concurrentDownloadLimitMB)*1024*1024,
 	)
 	// starting grpc server
 	grpcS := v.startGrpcService(volumeServer)
@@ -259,6 +262,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 
 		// Stop heartbeats
 		if !volumeServer.StopHeartbeat() {
+			volumeServer.SetStopping()
 			glog.V(0).Infof("stop send heartbeat and wait %d seconds until shutdown ...", *v.preStopSeconds)
 			time.Sleep(time.Duration(*v.preStopSeconds) * time.Second)
 		}

weed/filer/abstract_sql/abstract_sql_store.go

@@ -10,7 +10,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"strings"
 	"sync"
-	"time"
 )
 
 type SqlGenerator interface {
@@ -262,7 +261,7 @@ func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath util.Fu
 	return nil
 }
 
-func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath, limit int64) error {
+func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
 
 	db, bucket, shortPath, err := store.getTxOrDB(ctx, fullpath, true)
 	if err != nil {
@@ -280,22 +279,15 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat
 		}
 	}
 
-	for {
-		glog.V(4).Infof("delete %s SQL %s %d", string(shortPath), store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)))
-		res, err := db.ExecContext(ctx, store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)), string(shortPath), limit)
-		if err != nil {
-			return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err)
-		}
+	glog.V(4).Infof("delete %s SQL %s %d", string(shortPath), store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)))
+	res, err := db.ExecContext(ctx, store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)), string(shortPath))
+	if err != nil {
+		return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err)
+	}
 
-		rowCount, err := res.RowsAffected()
-		if err != nil {
-			return fmt.Errorf("deleteFolderChildren %s but no rows affected: %s", fullpath, err)
-		}
-		if rowCount < limit {
-			break
-		}
-		// to give the Galera Cluster a chance to breath
-		time.Sleep(time.Second)
+	_, err = res.RowsAffected()
+	if err != nil {
+		return fmt.Errorf("deleteFolderChildren %s but no rows affected: %s", fullpath, err)
 	}
 	return nil
 }

weed/filer/cassandra/cassandra_store.go

@@ -32,6 +32,7 @@ func (store *CassandraStore) Initialize(configuration util.Configuration, prefix
 		configuration.GetString(prefix+"username"),
 		configuration.GetString(prefix+"password"),
 		configuration.GetStringSlice(prefix+"superLargeDirectories"),
+		configuration.GetString(prefix+"localDC"),
 	)
 }
 
@@ -40,13 +41,19 @@ func (store *CassandraStore) isSuperLargeDirectory(dir string) (dirHash string,
 	return
 }
 
-func (store *CassandraStore) initialize(keyspace string, hosts []string, username string, password string, superLargeDirectories []string) (err error) {
+func (store *CassandraStore) initialize(keyspace string, hosts []string, username string, password string, superLargeDirectories []string, localDC string) (err error) {
 	store.cluster = gocql.NewCluster(hosts...)
 	if username != "" && password != "" {
 		store.cluster.Authenticator = gocql.PasswordAuthenticator{Username: username, Password: password}
 	}
 	store.cluster.Keyspace = keyspace
+	fallback := gocql.RoundRobinHostPolicy()
+	if localDC != "" {
+		fallback = gocql.DCAwareRoundRobinPolicy(localDC)
+	}
+	store.cluster.PoolConfig.HostSelectionPolicy = gocql.TokenAwareHostPolicy(fallback)
 	store.cluster.Consistency = gocql.LocalQuorum
+
 	store.session, err = store.cluster.CreateSession()
 	if err != nil {
 		glog.V(0).Infof("Failed to open cassandra store, hosts %v, keyspace %s", hosts, keyspace)
@@ -117,7 +124,7 @@ func (store *CassandraStore) FindEntry(ctx context.Context, fullpath util.FullPa
 	var data []byte
 	if err := store.session.Query(
 		"SELECT meta FROM filemeta WHERE directory=? AND name=?",
-		dir, name).Consistency(gocql.One).Scan(&data); err != nil {
+		dir, name).Scan(&data); err != nil {
 		if err != gocql.ErrNotFound {
 			return nil, filer_pb.ErrNotFound
 		}
@@ -154,7 +161,7 @@ func (store *CassandraStore) DeleteEntry(ctx context.Context, fullpath util.Full
 	return nil
 }
 
-func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath, limit int64) error {
+func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
 	if _, ok := store.isSuperLargeDirectory(string(fullpath)); ok {
 		return nil // filer.ErrUnsupportedSuperLargeDirectoryListing
 	}

weed/filer/cassandra/cassandra_store_kv.go

@@ -25,7 +25,7 @@ func (store *CassandraStore) KvGet(ctx context.Context, key []byte) (data []byte
 
 	if err := store.session.Query(
 		"SELECT meta FROM filemeta WHERE directory=? AND name=?",
-		dir, name).Consistency(gocql.One).Scan(&data); err != nil {
+		dir, name).Scan(&data); err != nil {
 		if err != gocql.ErrNotFound {
 			return nil, filer.ErrKvNotFound
 		}

weed/filer/elastic/v7/elastic_store.go

@@ -186,7 +186,7 @@ func (store *ElasticStore) deleteEntry(ctx context.Context, index, id string) (e
 	return fmt.Errorf("delete entry %v.", err)
 }
 
-func (store *ElasticStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath, limit int64) (err error) {
+func (store *ElasticStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
 	_, err = store.ListDirectoryEntries(ctx, fullpath, "", false, math.MaxInt32, func(entry *filer.Entry) bool {
 		if err := store.DeleteEntry(ctx, entry.FullPath); err != nil {
 			glog.Errorf("elastic delete %s: %v.", entry.FullPath, err)

weed/filer/entry.go

@@ -42,6 +42,7 @@ type Entry struct {
 	HardLinkId      HardLinkId
 	HardLinkCounter int32
 	Content         []byte
+	Remote          *filer_pb.RemoteEntry
 }
 
 func (entry *Entry) Size() uint64 {
@@ -56,20 +57,55 @@ func (entry *Entry) Timestamp() time.Time {
 	}
 }
 
+func (entry *Entry) ShallowClone() *Entry {
+	if entry == nil {
+		return nil
+	}
+	newEntry := &Entry{}
+	newEntry.FullPath = entry.FullPath
+	newEntry.Attr = entry.Attr
+	newEntry.Chunks = entry.Chunks
+	newEntry.Extended = entry.Extended
+	newEntry.HardLinkId = entry.HardLinkId
+	newEntry.HardLinkCounter = entry.HardLinkCounter
+	newEntry.Content = entry.Content
+	newEntry.Remote = entry.Remote
+
+	return newEntry
+}
+
 func (entry *Entry) ToProtoEntry() *filer_pb.Entry {
 	if entry == nil {
 		return nil
 	}
-	return &filer_pb.Entry{
-		Name:            entry.FullPath.Name(),
-		IsDirectory:     entry.IsDirectory(),
-		Attributes:      EntryAttributeToPb(entry),
-		Chunks:          entry.Chunks,
-		Extended:        entry.Extended,
-		HardLinkId:      entry.HardLinkId,
-		HardLinkCounter: entry.HardLinkCounter,
-		Content:         entry.Content,
+	message := &filer_pb.Entry{}
+	message.Name = entry.FullPath.Name()
+	entry.ToExistingProtoEntry(message)
+	return message
+}
+
+func (entry *Entry) ToExistingProtoEntry(message *filer_pb.Entry) {
+	if entry == nil {
+		return
 	}
+	message.IsDirectory = entry.IsDirectory()
+	message.Attributes = EntryAttributeToPb(entry)
+	message.Chunks = entry.Chunks
+	message.Extended = entry.Extended
+	message.HardLinkId = entry.HardLinkId
+	message.HardLinkCounter = entry.HardLinkCounter
+	message.Content = entry.Content
+	message.RemoteEntry = entry.Remote
+}
+
+func FromPbEntryToExistingEntry(message *filer_pb.Entry, fsEntry *Entry) {
+	fsEntry.Attr = PbToEntryAttribute(message.Attributes)
+	fsEntry.Chunks = message.Chunks
+	fsEntry.Extended = message.Extended
+	fsEntry.HardLinkId = HardLinkId(message.HardLinkId)
+	fsEntry.HardLinkCounter = message.HardLinkCounter
+	fsEntry.Content = message.Content
+	fsEntry.Remote = message.RemoteEntry
 }
 
 func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry {
@@ -83,26 +119,11 @@ func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry {
 	}
 }
 
-func (entry *Entry) Clone() *Entry {
-	return &Entry{
-		FullPath:        entry.FullPath,
-		Attr:            entry.Attr,
-		Chunks:          entry.Chunks,
-		Extended:        entry.Extended,
-		HardLinkId:      entry.HardLinkId,
-		HardLinkCounter: entry.HardLinkCounter,
-	}
-}
-
 func FromPbEntry(dir string, entry *filer_pb.Entry) *Entry {
-	return &Entry{
-		FullPath:        util.NewFullPath(dir, entry.Name),
-		Attr:            PbToEntryAttribute(entry.Attributes),
-		Chunks:          entry.Chunks,
-		HardLinkId:      HardLinkId(entry.HardLinkId),
-		HardLinkCounter: entry.HardLinkCounter,
-		Content:         entry.Content,
-	}
+	t := &Entry{}
+	t.FullPath = util.NewFullPath(dir, entry.Name)
+	FromPbEntryToExistingEntry(entry, t)
+	return t
 }
 
 func maxUint64(x, y uint64) uint64 {

weed/filer/entry_codec.go

@@ -12,14 +12,8 @@ import (
 )
 
 func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {
-	message := &filer_pb.Entry{
-		Attributes:      EntryAttributeToPb(entry),
-		Chunks:          entry.Chunks,
-		Extended:        entry.Extended,
-		HardLinkId:      entry.HardLinkId,
-		HardLinkCounter: entry.HardLinkCounter,
-		Content:         entry.Content,
-	}
+	message := &filer_pb.Entry{}
+	entry.ToExistingProtoEntry(message)
 	return proto.Marshal(message)
 }
 
@@ -31,15 +25,7 @@ func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error {
 		return fmt.Errorf("decoding value blob for %s: %v", entry.FullPath, err)
 	}
 
-	entry.Attr = PbToEntryAttribute(message.Attributes)
-
-	entry.Extended = message.Extended
-
-	entry.Chunks = message.Chunks
-
-	entry.HardLinkId = message.HardLinkId
-	entry.HardLinkCounter = message.HardLinkCounter
-	entry.Content = message.Content
+	FromPbEntryToExistingEntry(message, entry)
 
 	return nil
 }
@@ -129,6 +115,9 @@ func EqualEntry(a, b *Entry) bool {
 	if !bytes.Equal(a.Content, b.Content) {
 		return false
 	}
+	if !proto.Equal(a.Remote, b.Remote) {
+		return false
+	}
 	return true
 }
 

weed/filer/etcd/etcd_store.go

@@ -130,7 +130,7 @@ func (store *EtcdStore) DeleteEntry(ctx context.Context, fullpath weed_util.Full
 	return nil
 }
 
-func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath, limit int64) (err error) {
+func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
 	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
 
 	if _, err := store.client.Delete(ctx, string(directoryPrefix), clientv3.WithPrefix()); err != nil {

weed/filer/filechunk_manifest.go

@@ -16,7 +16,7 @@ import (
 )
 
 const (
-	ManifestBatch = 1000
+	ManifestBatch = 10000
 )
 
 func HasChunkManifest(chunks []*filer_pb.FileChunk) bool {
@@ -39,9 +39,14 @@ func SeparateManifestChunks(chunks []*filer_pb.FileChunk) (manifestChunks, nonMa
 	return
 }
 
-func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) {
+func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset, stopOffset int64) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) {
 	// TODO maybe parallel this
 	for _, chunk := range chunks {
+
+		if max(chunk.Offset, startOffset) >= min(chunk.Offset+int64(chunk.Size), stopOffset) {
+			continue
+		}
+
 		if !chunk.IsChunkManifest {
 			dataChunks = append(dataChunks, chunk)
 			continue
@@ -54,7 +59,7 @@ func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chun
 
 		manifestChunks = append(manifestChunks, chunk)
 		// recursive
-		dchunks, mchunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks)
+		dchunks, mchunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks, startOffset, stopOffset)
 		if subErr != nil {
 			return chunks, nil, subErr
 		}
@@ -127,6 +132,49 @@ func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool
 
 }
 
+func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) {
+
+	var shouldRetry bool
+	var totalWritten int
+
+	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
+		for _, urlString := range urlStrings {
+			var localProcesed int
+			shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
+				if totalWritten > localProcesed {
+					toBeSkipped := totalWritten - localProcesed
+					if len(data) <= toBeSkipped {
+						localProcesed += len(data)
+						return // skip if already processed
+					}
+					data = data[toBeSkipped:]
+					localProcesed += toBeSkipped
+				}
+				writer.Write(data)
+				localProcesed += len(data)
+				totalWritten += len(data)
+			})
+			if !shouldRetry {
+				break
+			}
+			if err != nil {
+				glog.V(0).Infof("read %s failed, err: %v", urlString, err)
+			} else {
+				break
+			}
+		}
+		if err != nil && shouldRetry {
+			glog.V(0).Infof("retry reading in %v", waitTime)
+			time.Sleep(waitTime)
+		} else {
+			break
+		}
+	}
+
+	return err
+
+}
+
 func MaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk) (chunks []*filer_pb.FileChunk, err error) {
 	return doMaybeManifestize(saveFunc, inputChunks, ManifestBatch, mergeIntoManifest)
 }

weed/filer/filechunks.go

@@ -53,7 +53,7 @@ func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) {
 
 func CompactFileChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
 
-	visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks)
+	visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks, 0, math.MaxInt64)
 
 	fileIds := make(map[string]bool)
 	for _, interval := range visibles {
@@ -72,11 +72,11 @@ func CompactFileChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks
 
 func MinusChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk, err error) {
 
-	aData, aMeta, aErr := ResolveChunkManifest(lookupFileIdFn, as)
+	aData, aMeta, aErr := ResolveChunkManifest(lookupFileIdFn, as, 0, math.MaxInt64)
 	if aErr != nil {
 		return nil, aErr
 	}
-	bData, bMeta, bErr := ResolveChunkManifest(lookupFileIdFn, bs)
+	bData, bMeta, bErr := ResolveChunkManifest(lookupFileIdFn, bs, 0, math.MaxInt64)
 	if bErr != nil {
 		return nil, bErr
 	}
@@ -117,7 +117,7 @@ func (cv *ChunkView) IsFullChunk() bool {
 
 func ViewFromChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (views []*ChunkView) {
 
-	visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks)
+	visibles, _ := NonOverlappingVisibleIntervals(lookupFileIdFn, chunks, offset, offset+size)
 
 	return ViewFromVisibleIntervals(visibles, offset, size)
 
@@ -221,9 +221,9 @@ func MergeIntoVisibles(visibles []VisibleInterval, chunk *filer_pb.FileChunk) (n
 
 // NonOverlappingVisibleIntervals translates the file chunk into VisibleInterval in memory
 // If the file chunk content is a chunk manifest
-func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (visibles []VisibleInterval, err error) {
+func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles []VisibleInterval, err error) {
 
-	chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks)
+	chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks, startOffset, stopOffset)
 
 	sort.Slice(chunks, func(i, j int) bool {
 		if chunks[i].Mtime == chunks[j].Mtime {

weed/filer/filechunks_test.go

@@ -90,7 +90,7 @@ func TestRandomFileChunksCompact(t *testing.T) {
 		}
 	}
 
-	visibles, _ := NonOverlappingVisibleIntervals(nil, chunks)
+	visibles, _ := NonOverlappingVisibleIntervals(nil, chunks, 0, math.MaxInt64)
 
 	for _, v := range visibles {
 		for x := v.start; x < v.stop; x++ {
@@ -227,7 +227,7 @@ func TestIntervalMerging(t *testing.T) {
 
 	for i, testcase := range testcases {
 		log.Printf("++++++++++ merged test case %d ++++++++++++++++++++", i)
-		intervals, _ := NonOverlappingVisibleIntervals(nil, testcase.Chunks)
+		intervals, _ := NonOverlappingVisibleIntervals(nil, testcase.Chunks, 0, math.MaxInt64)
 		for x, interval := range intervals {
 			log.Printf("test case %d, interval %d, start=%d, stop=%d, fileId=%s",
 				i, x, interval.start, interval.stop, interval.fileId)

weed/filer/filer.go

@@ -19,7 +19,6 @@ import (
 const (
 	LogFlushInterval = time.Minute
 	PaginationSize   = 1024
-	DeleteMaxRows    = 10000
 	FilerStoreId     = "filer.store.id"
 )
 
@@ -43,6 +42,7 @@ type Filer struct {
 	MetaAggregator      *MetaAggregator
 	Signature           int32
 	FilerConf           *FilerConf
+	RemoteStorage       *FilerRemoteStorage
 }
 
 func NewFiler(masters []string, grpcDialOption grpc.DialOption,
@@ -52,6 +52,7 @@ func NewFiler(masters []string, grpcDialOption grpc.DialOption,
 		fileIdDeletionQueue: util.NewUnboundedQueue(),
 		GrpcDialOption:      grpcDialOption,
 		FilerConf:           NewFilerConf(),
+		RemoteStorage:       NewFilerRemoteStorage(),
 	}
 	f.LocalMetaLogBuffer = log_buffer.NewLogBuffer("local", LogFlushInterval, f.logFlushFunc, notifyFn)
 	f.metaLogCollection = collection
@@ -208,7 +209,7 @@ func (f *Filer) ensureParentDirecotryEntry(ctx context.Context, entry *Entry, di
 			Attr: Attr{
 				Mtime:       now,
 				Crtime:      now,
-				Mode:        os.ModeDir | entry.Mode | 0110,
+				Mode:        os.ModeDir | entry.Mode | 0111,
 				Uid:         entry.Uid,
 				Gid:         entry.Gid,
 				Collection:  entry.Collection,

weed/filer/filer_buckets.go

@@ -3,6 +3,7 @@ package filer
 import (
 	"context"
 	"math"
+	"strings"
 	"sync"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -78,6 +79,9 @@ func (f *Filer) isBucket(entry *Entry) bool {
 	if parent != f.DirBucketsPath {
 		return false
 	}
+	if strings.HasPrefix(dirName, ".") {
+		return false
+	}
 
 	f.buckets.RLock()
 	defer f.buckets.RUnlock()

weed/filer/filer_conf.go

@@ -15,6 +15,7 @@ import (
 const (
 	DirectoryEtcRoot      = "/etc"
 	DirectoryEtcSeaweedFS = "/etc/seaweedfs"
+	DirectoryEtcRemote    = "/etc/remote"
 	FilerConfName         = "filer.conf"
 	IamConfigDirecotry    = "/etc/iam"
 	IamIdentityFile       = "identity.json"

weed/filer/filer_delete_entry.go

@@ -115,7 +115,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 
 	glog.V(3).Infof("deleting directory %v delete %d chunks: %v", entry.FullPath, len(chunks), shouldDeleteChunks)
 
-	if storeDeletionErr := f.Store.DeleteFolderChildren(ctx, entry.FullPath, DeleteMaxRows); storeDeletionErr != nil {
+	if storeDeletionErr := f.Store.DeleteFolderChildren(ctx, entry.FullPath); storeDeletionErr != nil {
 		return nil, nil, fmt.Errorf("filer store delete: %v", storeDeletionErr)
 	}
 

weed/filer/filer_deletion.go

@@ -10,9 +10,9 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )
 
-func LookupByMasterClientFn(masterClient *wdclient.MasterClient) func(vids []string) (map[string]operation.LookupResult, error) {
-	return func(vids []string) (map[string]operation.LookupResult, error) {
-		m := make(map[string]operation.LookupResult)
+func LookupByMasterClientFn(masterClient *wdclient.MasterClient) func(vids []string) (map[string]*operation.LookupResult, error) {
+	return func(vids []string) (map[string]*operation.LookupResult, error) {
+		m := make(map[string]*operation.LookupResult)
 		for _, vid := range vids {
 			locs, _ := masterClient.GetVidLocations(vid)
 			var locations []operation.Location
@@ -22,9 +22,9 @@ func LookupByMasterClientFn(masterClient *wdclient.MasterClient) func(vids []str
 					PublicUrl: loc.PublicUrl,
 				})
 			}
-			m[vid] = operation.LookupResult{
-				VolumeId:  vid,
-				Locations: locations,
+			m[vid] = &operation.LookupResult{
+				VolumeOrFileId: vid,
+				Locations:      locations,
 			}
 		}
 		return m, nil

weed/filer/filer_on_meta_event.go

@@ -12,6 +12,7 @@ import (
 // onMetadataChangeEvent is triggered after filer processed change events from local or remote filers
 func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) {
 	f.maybeReloadFilerConfiguration(event)
+	f.maybeReloadRemoteStorageConfigurationAndMapping(event)
 	f.onBucketEvents(event)
 }
 
@@ -24,10 +25,14 @@ func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
 	}
 	if f.DirBucketsPath == event.Directory {
 		if message.OldEntry == nil && message.NewEntry != nil {
-			f.Store.OnBucketCreation(message.NewEntry.Name)
+			if message.NewEntry.IsDirectory {
+				f.Store.OnBucketCreation(message.NewEntry.Name)
+			}
 		}
 		if message.OldEntry != nil && message.NewEntry == nil {
-			f.Store.OnBucketDeletion(message.OldEntry.Name)
+			if message.OldEntry.IsDirectory {
+				f.Store.OnBucketDeletion(message.OldEntry.Name)
+			}
 		}
 	}
 }
@@ -80,3 +85,16 @@ func (f *Filer) LoadFilerConf() {
 	}
 	f.FilerConf = fc
 }
+
+////////////////////////////////////
+// load and maintain remote storages
+////////////////////////////////////
+func (f *Filer) LoadRemoteStorageConfAndMapping() {
+	if err := f.RemoteStorage.LoadRemoteStorageConfigurationsAndMapping(f); err != nil {
+		glog.Errorf("read remote conf and mapping: %v", err)
+		return
+	}
+}
+func (f *Filer) maybeReloadRemoteStorageConfigurationAndMapping(event *filer_pb.SubscribeMetadataResponse) {
+	// FIXME add reloading
+}

weed/filer/filer_remote_storage.go

@@ -0,0 +1,229 @@
+package filer
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/remote_storage"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/golang/protobuf/proto"
+	"google.golang.org/grpc"
+	"math"
+	"strings"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/viant/ptrie"
+)
+
+const REMOTE_STORAGE_CONF_SUFFIX = ".conf"
+const REMOTE_STORAGE_MOUNT_FILE = "mount.mapping"
+
+type FilerRemoteStorage struct {
+	rules             ptrie.Trie
+	storageNameToConf map[string]*filer_pb.RemoteConf
+}
+
+func NewFilerRemoteStorage() (rs *FilerRemoteStorage) {
+	rs = &FilerRemoteStorage{
+		rules:             ptrie.New(),
+		storageNameToConf: make(map[string]*filer_pb.RemoteConf),
+	}
+	return rs
+}
+
+func (rs *FilerRemoteStorage) LoadRemoteStorageConfigurationsAndMapping(filer *Filer) (err error) {
+	// execute this on filer
+
+	limit := int64(math.MaxInt32)
+
+	entries, _, err := filer.ListDirectoryEntries(context.Background(), DirectoryEtcRemote, "", false, limit, "", "", "")
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			return nil
+		}
+		glog.Errorf("read remote storage %s: %v", DirectoryEtcRemote, err)
+		return
+	}
+
+	for _, entry := range entries {
+		if entry.Name() == REMOTE_STORAGE_MOUNT_FILE {
+			if err := rs.loadRemoteStorageMountMapping(entry.Content); err != nil {
+				return err
+			}
+			continue
+		}
+		if !strings.HasSuffix(entry.Name(), REMOTE_STORAGE_CONF_SUFFIX) {
+			return nil
+		}
+		conf := &filer_pb.RemoteConf{}
+		if err := proto.Unmarshal(entry.Content, conf); err != nil {
+			return fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, entry.Name(), err)
+		}
+		rs.storageNameToConf[conf.Name] = conf
+	}
+	return nil
+}
+
+func (rs *FilerRemoteStorage) loadRemoteStorageMountMapping(data []byte) (err error) {
+	mappings := &filer_pb.RemoteStorageMapping{}
+	if err := proto.Unmarshal(data, mappings); err != nil {
+		return fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, err)
+	}
+	for dir, storageLocation := range mappings.Mappings {
+		rs.mapDirectoryToRemoteStorage(util.FullPath(dir), storageLocation)
+	}
+	return nil
+}
+
+func (rs *FilerRemoteStorage) mapDirectoryToRemoteStorage(dir util.FullPath, loc *filer_pb.RemoteStorageLocation) {
+	rs.rules.Put([]byte(dir+"/"), loc)
+}
+
+func (rs *FilerRemoteStorage) FindMountDirectory(p util.FullPath) (mountDir util.FullPath, remoteLocation *filer_pb.RemoteStorageLocation) {
+	rs.rules.MatchPrefix([]byte(p), func(key []byte, value interface{}) bool {
+		mountDir = util.FullPath(string(key[:len(key)-1]))
+		remoteLocation = value.(*filer_pb.RemoteStorageLocation)
+		return true
+	})
+	return
+}
+
+func (rs *FilerRemoteStorage) FindRemoteStorageClient(p util.FullPath) (client remote_storage.RemoteStorageClient, remoteConf *filer_pb.RemoteConf, found bool) {
+	var storageLocation *filer_pb.RemoteStorageLocation
+	rs.rules.MatchPrefix([]byte(p), func(key []byte, value interface{}) bool {
+		storageLocation = value.(*filer_pb.RemoteStorageLocation)
+		return true
+	})
+
+	if storageLocation == nil {
+		found = false
+		return
+	}
+
+	return rs.GetRemoteStorageClient(storageLocation.Name)
+}
+
+func (rs *FilerRemoteStorage) GetRemoteStorageClient(storageName string) (client remote_storage.RemoteStorageClient, remoteConf *filer_pb.RemoteConf, found bool) {
+	remoteConf, found = rs.storageNameToConf[storageName]
+	if !found {
+		return
+	}
+
+	var err error
+	if client, err = remote_storage.GetRemoteStorage(remoteConf); err == nil {
+		found = true
+		return
+	}
+	return
+}
+
+func UnmarshalRemoteStorageMappings(oldContent []byte) (mappings *filer_pb.RemoteStorageMapping, err error) {
+	mappings = &filer_pb.RemoteStorageMapping{
+		Mappings: make(map[string]*filer_pb.RemoteStorageLocation),
+	}
+	if len(oldContent) > 0 {
+		if err = proto.Unmarshal(oldContent, mappings); err != nil {
+			glog.Warningf("unmarshal existing mappings: %v", err)
+		}
+	}
+	return
+}
+
+func AddRemoteStorageMapping(oldContent []byte, dir string, storageLocation *filer_pb.RemoteStorageLocation) (newContent []byte, err error) {
+	mappings, unmarshalErr := UnmarshalRemoteStorageMappings(oldContent)
+	if unmarshalErr != nil {
+		// skip
+	}
+
+	// set the new mapping
+	mappings.Mappings[dir] = storageLocation
+
+	if newContent, err = proto.Marshal(mappings); err != nil {
+		return oldContent, fmt.Errorf("marshal mappings: %v", err)
+	}
+
+	return
+}
+
+func RemoveRemoteStorageMapping(oldContent []byte, dir string) (newContent []byte, err error) {
+	mappings, unmarshalErr := UnmarshalRemoteStorageMappings(oldContent)
+	if unmarshalErr != nil {
+		return nil, unmarshalErr
+	}
+
+	// set the new mapping
+	delete(mappings.Mappings, dir)
+
+	if newContent, err = proto.Marshal(mappings); err != nil {
+		return oldContent, fmt.Errorf("marshal mappings: %v", err)
+	}
+
+	return
+}
+
+func ReadMountMappings(grpcDialOption grpc.DialOption, filerAddress string) (mappings *filer_pb.RemoteStorageMapping, readErr error) {
+	var oldContent []byte
+	if readErr = pb.WithFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)
+		return readErr
+	}); readErr != nil {
+		return nil, readErr
+	}
+
+	mappings, readErr = UnmarshalRemoteStorageMappings(oldContent)
+	if readErr != nil {
+		return nil, fmt.Errorf("unmarshal mappings: %v", readErr)
+	}
+
+	return
+}
+
+func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress string, storageName string) (conf *filer_pb.RemoteConf, readErr error) {
+	var oldContent []byte
+	if readErr = pb.WithFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, storageName+REMOTE_STORAGE_CONF_SUFFIX)
+		return readErr
+	}); readErr != nil {
+		return nil, readErr
+	}
+
+	// unmarshal storage configuration
+	conf = &filer_pb.RemoteConf{}
+	if unMarshalErr := proto.Unmarshal(oldContent, conf); unMarshalErr != nil {
+		readErr = fmt.Errorf("unmarshal %s/%s: %v", DirectoryEtcRemote, storageName+REMOTE_STORAGE_CONF_SUFFIX, unMarshalErr)
+		return
+	}
+
+	return
+}
+
+func DetectMountInfo(grpcDialOption grpc.DialOption, filerAddress string, dir string) (*filer_pb.RemoteStorageMapping, string, *filer_pb.RemoteStorageLocation, *filer_pb.RemoteConf, error) {
+
+	mappings, listErr := ReadMountMappings(grpcDialOption, filerAddress)
+	if listErr != nil {
+		return nil, "", nil, nil, listErr
+	}
+	if dir == "" {
+		return mappings, "", nil, nil, fmt.Errorf("need to specify '-dir' option")
+	}
+
+	var localMountedDir string
+	var remoteStorageMountedLocation *filer_pb.RemoteStorageLocation
+	for k, loc := range mappings.Mappings {
+		if strings.HasPrefix(dir, k) {
+			localMountedDir, remoteStorageMountedLocation = k, loc
+		}
+	}
+	if localMountedDir == "" {
+		return mappings, localMountedDir, remoteStorageMountedLocation, nil, fmt.Errorf("%s is not mounted", dir)
+	}
+
+	// find remote storage configuration
+	remoteStorageConf, err := ReadRemoteStorageConf(grpcDialOption, filerAddress, remoteStorageMountedLocation.Name)
+	if err != nil {
+		return mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, err
+	}
+
+	return mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, nil
+}

weed/filer/filer_remote_storage_test.go

@@ -0,0 +1,34 @@
+package filer
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestFilerRemoteStorage_FindRemoteStorageClient(t *testing.T) {
+	conf := &filer_pb.RemoteConf{
+		Name: "s7",
+		Type: "s3",
+	}
+	rs := NewFilerRemoteStorage()
+	rs.storageNameToConf[conf.Name] = conf
+
+	rs.mapDirectoryToRemoteStorage("/a/b/c", &filer_pb.RemoteStorageLocation{
+		Name:   "s7",
+		Bucket: "some",
+		Path:   "/dir",
+	})
+
+	_, _, found := rs.FindRemoteStorageClient("/a/b/c/d/e/f")
+	assert.Equal(t, true, found, "find storage client")
+
+	_, _, found2 := rs.FindRemoteStorageClient("/a/b")
+	assert.Equal(t, false, found2, "should not find storage client")
+
+	_, _, found3 := rs.FindRemoteStorageClient("/a/b/c")
+	assert.Equal(t, false, found3, "should not find storage client")
+
+	_, _, found4 := rs.FindRemoteStorageClient("/a/b/cc")
+	assert.Equal(t, false, found4, "should not find storage client")
+}

weed/filer/filer_search.go

@@ -3,6 +3,7 @@ package filer
 import (
 	"context"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"math"
 	"path/filepath"
 	"strings"
 )
@@ -27,6 +28,10 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, start
 		return true
 	})
 
+	if limit == math.MaxInt64 {
+		limit = math.MaxInt64 - 1
+	}
+
 	hasMore = int64(len(entries)) >= limit+1
 	if hasMore {
 		entries = entries[:limit]

weed/filer/filerstore.go

@@ -25,7 +25,7 @@ type FilerStore interface {
 	// err == filer_pb.ErrNotFound if not found
 	FindEntry(context.Context, util.FullPath) (entry *Entry, err error)
 	DeleteEntry(context.Context, util.FullPath) (err error)
-	DeleteFolderChildren(context.Context, util.FullPath, int64) (err error)
+	DeleteFolderChildren(context.Context, util.FullPath) (err error)
 	ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)
 	ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)
 

weed/filer/filerstore_translate_path.go

@@ -100,10 +100,10 @@ func (t *FilerStorePathTranlator) DeleteOneEntry(ctx context.Context, existingEn
 	return t.actualStore.DeleteEntry(ctx, existingEntry.FullPath)
 }
 
-func (t *FilerStorePathTranlator) DeleteFolderChildren(ctx context.Context, fp util.FullPath, limit int64) (err error) {
+func (t *FilerStorePathTranlator) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
 	newFullPath := t.translatePath(fp)
 
-	return t.actualStore.DeleteFolderChildren(ctx, newFullPath, limit)
+	return t.actualStore.DeleteFolderChildren(ctx, newFullPath)
 }
 
 func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) {

weed/filer/filerstore_wrapper.go

@@ -213,7 +213,7 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry
 	return actualStore.DeleteEntry(ctx, existingEntry.FullPath)
 }
 
-func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath, limit int64) (err error) {
+func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
 	actualStore := fsw.getActualStore(fp + "/")
 	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "deleteFolderChildren").Inc()
 	start := time.Now()
@@ -222,7 +222,7 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.
 	}()
 
 	glog.V(4).Infof("DeleteFolderChildren %s", fp)
-	return actualStore.DeleteFolderChildren(ctx, fp, limit)
+	return actualStore.DeleteFolderChildren(ctx, fp)
 }
 
 func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) {

weed/filer/hbase/hbase_store.go

@@ -109,7 +109,7 @@ func (store *HbaseStore) DeleteEntry(ctx context.Context, path util.FullPath) (e
 	return store.doDelete(ctx, store.cfMetaDir, []byte(path))
 }
 
-func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.FullPath, limit int64) (err error) {
+func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.FullPath) (err error) {
 
 	family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}}
 	expectedPrefix := []byte(path.Child(""))

weed/filer/leveldb/leveldb_store.go

@@ -136,7 +136,7 @@ func (store *LevelDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.F
 	return nil
 }
 
-func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath, limit int64) (err error) {
+func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
 
 	batch := new(leveldb.Batch)
 

weed/filer/leveldb2/leveldb2_store.go

@@ -5,13 +5,15 @@ import (
 	"context"
 	"crypto/md5"
 	"fmt"
-	"github.com/syndtr/goleveldb/leveldb"
-	leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
 	"io"
 	"os"
 
+	"github.com/syndtr/goleveldb/leveldb"
+	leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
+
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@@ -47,6 +49,7 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
 		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
 		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
 		CompactionTableSizeMultiplier: 4,
+		Filter:                        filter.NewBloomFilter(8), // false positive rate 0.02
 	}
 
 	for d := 0; d < dbCount; d++ {
@@ -144,7 +147,7 @@ func (store *LevelDB2Store) DeleteEntry(ctx context.Context, fullpath weed_util.
 	return nil
 }
 
-func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath, limit int64) (err error) {
+func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
 	directoryPrefix, partitionId := genDirectoryKeyPrefix(fullpath, "", store.dbCount)
 
 	batch := new(leveldb.Batch)

weed/filer/leveldb3/leveldb3_store.go

@@ -5,15 +5,17 @@ import (
 	"context"
 	"crypto/md5"
 	"fmt"
-	"github.com/syndtr/goleveldb/leveldb"
-	leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
 	"io"
 	"os"
 	"strings"
 	"sync"
 
+	"github.com/syndtr/goleveldb/leveldb"
+	leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
+
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@@ -62,17 +64,19 @@ func (store *LevelDB3Store) initialize(dir string) (err error) {
 }
 
 func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) {
-
+	bloom := filter.NewBloomFilter(8) // false positive rate 0.02
 	opts := &opt.Options{
 		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
 		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
 		CompactionTableSizeMultiplier: 4,
+		Filter:                        bloom,
 	}
 	if name != DEFAULT {
 		opts = &opt.Options{
 			BlockCacheCapacity:            4 * 1024 * 1024, // default value is 8MiB
 			WriteBuffer:                   2 * 1024 * 1024, // default value is 4MiB
 			CompactionTableSizeMultiplier: 4,
+			Filter:                        bloom,
 		}
 	}
 
@@ -245,7 +249,7 @@ func (store *LevelDB3Store) DeleteEntry(ctx context.Context, fullpath weed_util.
 	return nil
 }
 
-func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath, limit int64) (err error) {
+func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
 
 	db, bucket, shortPath, err := store.findDB(fullpath, true)
 	if err != nil {

weed/filer/mongodb/mongodb_store.go

@@ -167,7 +167,7 @@ func (store *MongodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPa
 	return nil
 }
 
-func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath, limit int64) error {
+func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
 
 	where := bson.M{"directory": fullpath}
 	_, err := store.connect.Database(store.database).Collection(store.collectionName).DeleteMany(ctx, where)

weed/filer/mysql/mysql_sql_gen.go

@@ -38,7 +38,7 @@ func (gen *SqlGenMysql) GetSqlDelete(tableName string) string {
 }
 
 func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(tableName string) string {
-	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=? LIMIT ?", tableName)
+	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=?", tableName)
 }
 
 func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {

weed/filer/read_remote.go

@@ -0,0 +1,49 @@
+package filer
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+func (entry *Entry) IsInRemoteOnly() bool {
+	return len(entry.Chunks) == 0 && entry.Remote != nil && entry.Remote.RemoteSize > 0
+}
+
+func (f *Filer) ReadRemote(entry *Entry, offset int64, size int64) (data []byte, err error) {
+	client, _, found := f.RemoteStorage.GetRemoteStorageClient(entry.Remote.StorageName)
+	if !found {
+		return nil, fmt.Errorf("remote storage %v not found", entry.Remote.StorageName)
+	}
+
+	mountDir, remoteLoation := f.RemoteStorage.FindMountDirectory(entry.FullPath)
+
+	sourceLoc := MapFullPathToRemoteStorageLocation(mountDir, remoteLoation, entry.FullPath)
+
+	return client.ReadFile(sourceLoc, offset, size)
+}
+
+func MapFullPathToRemoteStorageLocation(localMountedDir util.FullPath, remoteMountedLocation *filer_pb.RemoteStorageLocation, fp util.FullPath) *filer_pb.RemoteStorageLocation {
+	remoteLocation := &filer_pb.RemoteStorageLocation{
+		Name:   remoteMountedLocation.Name,
+		Bucket: remoteMountedLocation.Bucket,
+		Path:   remoteMountedLocation.Path,
+	}
+	remoteLocation.Path = string(util.FullPath(remoteLocation.Path).Child(string(fp)[len(localMountedDir):]))
+	return remoteLocation
+}
+
+func MapRemoteStorageLocationPathToFullPath(localMountedDir util.FullPath, remoteMountedLocation *filer_pb.RemoteStorageLocation, remoteLocationPath string)(fp util.FullPath)  {
+	return localMountedDir.Child(remoteLocationPath[len(remoteMountedLocation.Path):])
+}
+
+func DownloadToLocal(filerClient filer_pb.FilerClient, remoteConf *filer_pb.RemoteConf, remoteLocation *filer_pb.RemoteStorageLocation, parent util.FullPath, entry *filer_pb.Entry) error {
+	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		_, err := client.DownloadToLocal(context.Background(), &filer_pb.DownloadToLocalRequest{
+			Directory: string(parent),
+			Name:      entry.Name,
+		})
+		return err
+	})
+}

weed/filer/read_write.go

@@ -2,13 +2,9 @@ package filer
 
 import (
 	"bytes"
-	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
-	"io/ioutil"
 	"math"
-	"net/http"
 	"time"
 )
 
@@ -31,50 +27,17 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed
 
 }
 
-func ReadContent(filerAddress string, dir, name string) ([]byte, error) {
-
-	target := fmt.Sprintf("http://%s%s/%s", filerAddress, dir, name)
-
-	data, _, err := util.Get(target)
-
-	return data, err
-}
-
-func SaveAs(host string, port int, dir, name string, contentType string, byteBuffer *bytes.Buffer) error {
-	var target string
-	if port == 0 {
-		target = fmt.Sprintf("http://%s%s/%s", host, dir, name)
-	} else {
-		target = fmt.Sprintf("http://%s:%d%s/%s", host, port, dir, name)
+func ReadInsideFiler(filerClient filer_pb.SeaweedFilerClient, dir, name string) (content []byte, err error) {
+	request := &filer_pb.LookupDirectoryEntryRequest{
+		Directory: dir,
+		Name:      name,
 	}
-
-	// set the HTTP method, url, and request body
-	req, err := http.NewRequest(http.MethodPut, target, byteBuffer)
+	respLookupEntry, err := filer_pb.LookupEntry(filerClient, request)
 	if err != nil {
-		return err
+		return
 	}
-
-	// set the request header Content-Type for json
-	if contentType != "" {
-		req.Header.Set("Content-Type", contentType)
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer util.CloseResponse(resp)
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	if resp.StatusCode >= 400 {
-		return fmt.Errorf("%s: %s %v", target, resp.Status, string(b))
-	}
-
-	return nil
-
+	content = respLookupEntry.Entry.Content
+	return
 }
 
 func SaveInsideFiler(client filer_pb.SeaweedFilerClient, dir, name string, content []byte) error {

weed/filer/redis/universal_redis_store.go

@@ -107,7 +107,7 @@ func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath util
 	return nil
 }
 
-func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath, limit int64) (err error) {
+func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
 
 	members, err := store.Client.SMembers(ctx, genDirectoryListKey(string(fullpath))).Result()
 	if err != nil {

weed/filer/redis2/universal_redis_store.go

@@ -127,7 +127,7 @@ func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath uti
 	return nil
 }
 
-func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath, limit int64) (err error) {
+func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
 
 	if store.isSuperLargeDirectory(string(fullpath)) {
 		return nil

weed/filer/rocksdb/rocksdb_store.go

@@ -24,18 +24,21 @@ func init() {
 
 type options struct {
 	opt *gorocksdb.Options
+	bto *gorocksdb.BlockBasedTableOptions
 	ro  *gorocksdb.ReadOptions
 	wo  *gorocksdb.WriteOptions
 }
 
 func (opt *options) init() {
 	opt.opt = gorocksdb.NewDefaultOptions()
+	opt.bto = gorocksdb.NewDefaultBlockBasedTableOptions()
 	opt.ro = gorocksdb.NewDefaultReadOptions()
 	opt.wo = gorocksdb.NewDefaultWriteOptions()
 }
 
 func (opt *options) close() {
 	opt.opt.Destroy()
+	opt.bto.Destroy()
 	opt.ro.Destroy()
 	opt.wo.Destroy()
 }
@@ -69,6 +72,11 @@ func (store *RocksDBStore) initialize(dir string) (err error) {
 	store.opt.SetCompactionFilter(NewTTLFilter())
 	// store.opt.SetMaxBackgroundCompactions(2)
 
+	// https://github.com/tecbot/gorocksdb/issues/132
+	store.bto.SetFilterPolicy(gorocksdb.NewBloomFilterFull(8))
+	store.opt.SetBlockBasedTableFactory(store.bto)
+	// store.opt.EnableStatistics()
+
 	store.db, err = gorocksdb.OpenDb(store.opt, dir)
 
 	return
@@ -148,7 +156,7 @@ func (store *RocksDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.F
 	return nil
 }
 
-func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath, limit int64) (err error) {
+func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
 	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
 
 	batch := gorocksdb.NewWriteBatch()

weed/filer/s3iam_conf.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
 	"github.com/golang/protobuf/jsonpb"
+	"github.com/golang/protobuf/proto"
 	"io"
 )
 
@@ -14,7 +15,7 @@ func ParseS3ConfigurationFromBytes(content []byte, config *iam_pb.S3ApiConfigura
 	return nil
 }
 
-func S3ConfigurationToText(writer io.Writer, config *iam_pb.S3ApiConfiguration) error {
+func ProtoToText(writer io.Writer, config proto.Message) error {
 
 	m := jsonpb.Marshaler{
 		EmitDefaults: false,

weed/filer/s3iam_conf_test.go

@@ -44,7 +44,7 @@ func TestS3Conf(t *testing.T) {
 		},
 	}
 	var buf bytes.Buffer
-	err := S3ConfigurationToText(&buf, s3Conf)
+	err := ProtoToText(&buf, s3Conf)
 	assert.Equal(t, err, nil)
 	s3ConfSaved := &iam_pb.S3ApiConfiguration{}
 	err = ParseS3ConfigurationFromBytes(buf.Bytes(), s3ConfSaved)

weed/filer/sqlite/sqlite_store_unsupported.go

@@ -1,4 +1,4 @@
-// +build !linux,!darwin,!windows
+// +build !linux,!darwin,!windows,!s390,!ppc64le,!mips64
 
 // limited GOOS due to modernc.org/libc/unistd
 

weed/filer/stream.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"sort"
 	"strings"
 	"time"
 
@@ -15,7 +16,50 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )
 
-func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
+func HasData(entry *filer_pb.Entry) bool {
+
+	if len(entry.Content) > 0 {
+		return true
+	}
+
+	return len(entry.Chunks) > 0
+}
+
+func IsSameData(a, b *filer_pb.Entry) bool {
+
+	if len(a.Content) > 0 || len(b.Content) > 0 {
+		return bytes.Equal(a.Content, b.Content)
+	}
+
+	return isSameChunks(a.Chunks, b.Chunks)
+}
+
+func isSameChunks(a, b []*filer_pb.FileChunk) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	sort.Slice(a, func(i, j int) bool {
+		return strings.Compare(a[i].ETag, a[j].ETag) < 0
+	})
+	sort.Slice(b, func(i, j int) bool {
+		return strings.Compare(b[i].ETag, b[j].ETag) < 0
+	})
+	for i := 0; i < len(a); i++ {
+		if a[i].ETag != b[i].ETag {
+			return false
+		}
+	}
+	return true
+}
+
+func NewFileReader(filerClient filer_pb.FilerClient, entry *filer_pb.Entry) io.Reader {
+	if len(entry.Content) > 0 {
+		return bytes.NewReader(entry.Content)
+	}
+	return NewChunkStreamReader(filerClient, entry.Chunks)
+}
+
+func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
 
 	glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks)
 	chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
@@ -39,18 +83,12 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c
 
 		urlStrings := fileId2Url[chunkView.FileId]
 		start := time.Now()
-		data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
+		err := retriedStreamFetchChunkData(writer, urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size))
 		stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds())
 		if err != nil {
 			stats.FilerRequestCounter.WithLabelValues("chunkDownloadError").Inc()
 			return fmt.Errorf("read chunk: %v", err)
 		}
-
-		_, err = w.Write(data)
-		if err != nil {
-			stats.FilerRequestCounter.WithLabelValues("chunkDownloadedError").Inc()
-			return fmt.Errorf("write chunk: %v", err)
-		}
 		stats.FilerRequestCounter.WithLabelValues("chunkDownload").Inc()
 	}
 
@@ -88,16 +126,37 @@ func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk)
 
 // ----------------  ChunkStreamReader ----------------------------------
 type ChunkStreamReader struct {
-	chunkViews   []*ChunkView
-	logicOffset  int64
-	buffer       []byte
-	bufferOffset int64
-	bufferPos    int
-	chunkIndex   int
-	lookupFileId wdclient.LookupFileIdFunctionType
+	chunkViews         []*ChunkView
+	totalSize          int64
+	logicOffset        int64
+	buffer             []byte
+	bufferOffset       int64
+	bufferPos          int
+	nextChunkViewIndex int
+	lookupFileId       wdclient.LookupFileIdFunctionType
 }
 
 var _ = io.ReadSeeker(&ChunkStreamReader{})
+var _ = io.ReaderAt(&ChunkStreamReader{})
+
+func doNewChunkStreamReader(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
+
+	chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
+	sort.Slice(chunkViews, func(i, j int) bool {
+		return chunkViews[i].LogicOffset < chunkViews[j].LogicOffset
+	})
+
+	var totalSize int64
+	for _, chunk := range chunkViews {
+		totalSize += int64(chunk.Size)
+	}
+
+	return &ChunkStreamReader{
+		chunkViews:   chunkViews,
+		lookupFileId: lookupFileIdFn,
+		totalSize:    totalSize,
+	}
+}
 
 func NewChunkStreamReaderFromFiler(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
 
@@ -105,39 +164,40 @@ func NewChunkStreamReaderFromFiler(masterClient *wdclient.MasterClient, chunks [
 		return masterClient.LookupFileId(fileId)
 	}
 
-	chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
-
-	return &ChunkStreamReader{
-		chunkViews:   chunkViews,
-		lookupFileId: lookupFileIdFn,
-	}
+	return doNewChunkStreamReader(lookupFileIdFn, chunks)
 }
 
 func NewChunkStreamReader(filerClient filer_pb.FilerClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
 
 	lookupFileIdFn := LookupFn(filerClient)
 
-	chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
+	return doNewChunkStreamReader(lookupFileIdFn, chunks)
+}
 
-	return &ChunkStreamReader{
-		chunkViews:   chunkViews,
-		lookupFileId: lookupFileIdFn,
+func (c *ChunkStreamReader) ReadAt(p []byte, off int64) (n int, err error) {
+	if err = c.prepareBufferFor(off); err != nil {
+		return
 	}
+	c.logicOffset = off
+	return c.Read(p)
 }
 
 func (c *ChunkStreamReader) Read(p []byte) (n int, err error) {
 	for n < len(p) {
 		if c.isBufferEmpty() {
-			if c.chunkIndex >= len(c.chunkViews) {
+			if c.nextChunkViewIndex >= len(c.chunkViews) {
 				return n, io.EOF
 			}
-			chunkView := c.chunkViews[c.chunkIndex]
-			c.fetchChunkToBuffer(chunkView)
-			c.chunkIndex++
+			chunkView := c.chunkViews[c.nextChunkViewIndex]
+			if err = c.fetchChunkToBuffer(chunkView); err != nil {
+				return
+			}
+			c.nextChunkViewIndex++
 		}
 		t := copy(p[n:], c.buffer[c.bufferPos:])
 		c.bufferPos += t
 		n += t
+		c.logicOffset += int64(t)
 	}
 	return
 }
@@ -148,38 +208,70 @@ func (c *ChunkStreamReader) isBufferEmpty() bool {
 
 func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error) {
 
-	var totalSize int64
-	for _, chunk := range c.chunkViews {
-		totalSize += int64(chunk.Size)
-	}
-
 	var err error
 	switch whence {
 	case io.SeekStart:
 	case io.SeekCurrent:
-		offset += c.bufferOffset + int64(c.bufferPos)
+		offset += c.logicOffset
 	case io.SeekEnd:
-		offset = totalSize + offset
+		offset = c.totalSize + offset
 	}
-	if offset > totalSize {
+	if offset > c.totalSize {
 		err = io.ErrUnexpectedEOF
+	} else {
+		c.logicOffset = offset
 	}
 
-	for i, chunk := range c.chunkViews {
-		if chunk.LogicOffset <= offset && offset < chunk.LogicOffset+int64(chunk.Size) {
-			if c.isBufferEmpty() || c.bufferOffset != chunk.LogicOffset {
-				c.fetchChunkToBuffer(chunk)
-				c.chunkIndex = i + 1
-				break
-			}
-		}
-	}
-	c.bufferPos = int(offset - c.bufferOffset)
-
 	return offset, err
 
 }
 
+func (c *ChunkStreamReader) prepareBufferFor(offset int64) (err error) {
+	// stay in the same chunk
+	if !c.isBufferEmpty() {
+		if c.bufferOffset <= offset && offset < c.bufferOffset+int64(len(c.buffer)) {
+			c.bufferPos = int(offset - c.bufferOffset)
+			return nil
+		}
+	}
+
+	// need to seek to a different chunk
+	currentChunkIndex := sort.Search(len(c.chunkViews), func(i int) bool {
+		return offset < c.chunkViews[i].LogicOffset
+	})
+	if currentChunkIndex == len(c.chunkViews) {
+		// not found
+		if c.chunkViews[0].LogicOffset <= offset {
+			currentChunkIndex = 0
+		} else if c.chunkViews[len(c.chunkViews)-1].LogicOffset <= offset {
+			currentChunkIndex = len(c.chunkViews) -1
+		} else {
+			return io.EOF
+		}
+	} else if currentChunkIndex > 0 {
+		if c.chunkViews[currentChunkIndex-1].LogicOffset <= offset {
+			currentChunkIndex -= 1
+		} else {
+			return fmt.Errorf("unexpected1 offset %d", offset)
+		}
+	} else {
+		return fmt.Errorf("unexpected2 offset %d", offset)
+	}
+
+	// positioning within the new chunk
+	chunk := c.chunkViews[currentChunkIndex]
+	if chunk.LogicOffset <= offset && offset < chunk.LogicOffset+int64(chunk.Size) {
+		if c.isBufferEmpty() || c.bufferOffset != chunk.LogicOffset {
+			if err = c.fetchChunkToBuffer(chunk); err != nil {
+				return
+			}
+			c.nextChunkViewIndex = currentChunkIndex + 1
+		}
+		c.bufferPos = int(offset - c.bufferOffset)
+	}
+	return
+}
+
 func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
 	urlStrings, err := c.lookupFileId(chunkView.FileId)
 	if err != nil {

weed/filesys/dir.go

@@ -141,7 +141,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 	var node fs.Node
 	if isDirectory {
 		node = dir.newDirectory(util.NewFullPath(dir.FullPath(), req.Name))
-		return node, nil, nil
+		return node, node, nil
 	}
 
 	node = dir.newFile(req.Name)

weed/filesys/dir_rename.go

@@ -2,6 +2,9 @@ package filesys
 
 import (
 	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"math"
 
 	"github.com/seaweedfs/fuse"
 	"github.com/seaweedfs/fuse/fs"
@@ -37,6 +40,7 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 			OldName:      req.OldName,
 			NewDirectory: newDir.FullPath(),
 			NewName:      req.NewName,
+			Signatures:   []int32{dir.wfs.signature},
 		}
 
 		_, err := client.AtomicRenameEntry(ctx, request)
@@ -53,46 +57,118 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 		return fuse.EIO
 	}
 
-	// TODO: replicate renaming logic on filer
-	if err := dir.wfs.metaCache.DeleteEntry(context.Background(), oldPath); err != nil {
-		glog.V(0).Infof("dir Rename delete local %s => %s : %v", oldPath, newPath, err)
+	err = dir.moveEntry(context.Background(), util.FullPath(dir.FullPath()), oldEntry, util.FullPath(newDir.FullPath()), req.NewName)
+	if err != nil {
+		glog.V(0).Infof("dir local Rename %s => %s : %v", oldPath, newPath, err)
 		return fuse.EIO
 	}
-	oldEntry.FullPath = newPath
-	if err := dir.wfs.metaCache.InsertEntry(context.Background(), oldEntry); err != nil {
+
+	return nil
+}
+
+func (dir *Dir) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string) error {
+
+	oldName := entry.Name()
+
+	oldPath := oldParent.Child(oldName)
+	newPath := newParent.Child(newName)
+	if err := dir.moveSelfEntry(ctx, oldParent, entry, newParent, newName, func() error {
+
+		oldFsNode := NodeWithId(oldPath.AsInode())
+		newFsNode := NodeWithId(newPath.AsInode())
+		newDirNode, found := dir.wfs.Server.FindInternalNode(NodeWithId(newParent.AsInode()))
+		var newDir *Dir
+		if found {
+			newDir = newDirNode.(*Dir)
+		}
+		dir.wfs.Server.InvalidateInternalNode(oldFsNode, newFsNode, func(internalNode fs.Node) {
+			if file, ok := internalNode.(*File); ok {
+				glog.V(4).Infof("internal file node %s", oldParent.Child(oldName))
+				file.Name = newName
+				file.id = uint64(newFsNode)
+				if found {
+					file.dir = newDir
+				}
+			}
+			if dir, ok := internalNode.(*Dir); ok {
+				glog.V(4).Infof("internal dir node %s", oldParent.Child(oldName))
+				dir.name = newName
+				dir.id = uint64(newFsNode)
+				if found {
+					dir.parent = newDir
+				}
+			}
+		})
+
+		// change file handle
+		inodeId := oldPath.AsInode()
+		dir.wfs.handlesLock.Lock()
+		if existingHandle, found := dir.wfs.handles[inodeId]; found && existingHandle == nil {
+			glog.V(4).Infof("opened file handle %s => %s", oldPath, newPath)
+			delete(dir.wfs.handles, inodeId)
+			dir.wfs.handles[newPath.AsInode()] = existingHandle
+		}
+		dir.wfs.handlesLock.Unlock()
+
+		if entry.IsDirectory() {
+			if err := dir.moveFolderSubEntries(ctx, oldParent, oldName, newParent, newName); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		return fmt.Errorf("fail to move %s => %s: %v", oldPath, newPath, err)
+	}
+
+	return nil
+}
+
+func (dir *Dir) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, oldName string, newParent util.FullPath, newName string) error {
+
+	currentDirPath := oldParent.Child(oldName)
+	newDirPath := newParent.Child(newName)
+
+	glog.V(1).Infof("moving folder %s => %s", currentDirPath, newDirPath)
+
+	var moveErr error
+	listErr := dir.wfs.metaCache.ListDirectoryEntries(ctx, currentDirPath, "", false, int64(math.MaxInt32), func(item *filer.Entry) bool {
+		moveErr = dir.moveEntry(ctx, currentDirPath, item, newDirPath, item.Name())
+		if moveErr != nil {
+			return false
+		}
+		return true
+	})
+	if listErr != nil {
+		return listErr
+	}
+	if moveErr != nil {
+		return moveErr
+	}
+
+	return nil
+}
+
+func (dir *Dir) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer.Entry, newParent util.FullPath, newName string, moveFolderSubEntries func() error) error {
+
+	newPath := newParent.Child(newName)
+	oldPath := oldParent.Child(entry.Name())
+
+	entry.FullPath = newPath
+	if err := dir.wfs.metaCache.InsertEntry(ctx, entry); err != nil {
 		glog.V(0).Infof("dir Rename insert local %s => %s : %v", oldPath, newPath, err)
 		return fuse.EIO
 	}
 
-	oldFsNode := NodeWithId(oldPath.AsInode())
-	newFsNode := NodeWithId(newPath.AsInode())
-	dir.wfs.Server.InvalidateInternalNode(oldFsNode, newFsNode, func(internalNode fs.Node) {
-		if file, ok := internalNode.(*File); ok {
-			glog.V(4).Infof("internal file node %s", file.Name)
-			file.Name = req.NewName
-			file.id = uint64(newFsNode)
-			file.dir = newDir
+	if moveFolderSubEntries != nil {
+		if moveChildrenErr := moveFolderSubEntries(); moveChildrenErr != nil {
+			return moveChildrenErr
 		}
-		if dir, ok := internalNode.(*Dir); ok {
-			glog.V(4).Infof("internal dir node %s", dir.name)
-			dir.name = req.NewName
-			dir.id = uint64(newFsNode)
-			dir.parent = newDir
-		}
-	})
-
-	// change file handle
-	dir.wfs.handlesLock.Lock()
-	defer dir.wfs.handlesLock.Unlock()
-	inodeId := oldPath.AsInode()
-	existingHandle, found := dir.wfs.handles[inodeId]
-	glog.V(4).Infof("has open filehandle %s: %v", oldPath, found)
-	if !found || existingHandle == nil {
-		return nil
 	}
-	glog.V(4).Infof("opened filehandle %s => %s", oldPath, newPath)
-	delete(dir.wfs.handles, inodeId)
-	dir.wfs.handles[newPath.AsInode()] = existingHandle
+
+	if err := dir.wfs.metaCache.DeleteEntry(ctx, oldPath); err != nil {
+		glog.V(0).Infof("dir Rename delete local %s => %s : %v", oldPath, newPath, err)
+		return fuse.EIO
+	}
 
 	return nil
 }

weed/filesys/dirty_pages_temp_file.go

@@ -97,7 +97,7 @@ func (pages *TempFileDirtyPages) saveExistingPagesToStorage() {
 
 	for _, list := range pages.writtenIntervals.lists {
 		listStopOffset := list.Offset() + list.Size()
-		for uploadedOffset:=int64(0); uploadedOffset < listStopOffset; uploadedOffset += pageSize {
+		for uploadedOffset := int64(0); uploadedOffset < listStopOffset; uploadedOffset += pageSize {
 			start, stop := max(list.Offset(), uploadedOffset), min(listStopOffset, uploadedOffset+pageSize)
 			if start >= stop {
 				continue

weed/filesys/dirty_pages_temp_interval.go

@@ -54,7 +54,7 @@ func (list *WrittenIntervalLinkedList) ReadData(buf []byte, start, stop int64) {
 		nodeStart, nodeStop := max(start, t.DataOffset), min(stop, t.DataOffset+t.Size)
 		if nodeStart < nodeStop {
 			// glog.V(4).Infof("copying start=%d stop=%d t=[%d,%d) => bufSize=%d nodeStart=%d, nodeStop=%d", start, stop, t.DataOffset, t.DataOffset+t.Size, len(buf), nodeStart, nodeStop)
-			list.tempFile.ReadAt(buf[nodeStart-start:nodeStop-start], t.TempOffset + nodeStart - t.DataOffset)
+			list.tempFile.ReadAt(buf[nodeStart-start:nodeStop-start], t.TempOffset+nodeStart-t.DataOffset)
 		}
 
 		if t.Next == nil {

weed/filesys/file.go

@@ -144,17 +144,17 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		file.dirtyMetadata = true
 	}
 
-	if req.Valid.Mode() {
+	if req.Valid.Mode() && entry.Attributes.FileMode != uint32(req.Mode) {
 		entry.Attributes.FileMode = uint32(req.Mode)
 		file.dirtyMetadata = true
 	}
 
-	if req.Valid.Uid() {
+	if req.Valid.Uid() && entry.Attributes.Uid != req.Uid {
 		entry.Attributes.Uid = req.Uid
 		file.dirtyMetadata = true
 	}
 
-	if req.Valid.Gid() {
+	if req.Valid.Gid() && entry.Attributes.Gid != req.Gid {
 		entry.Attributes.Gid = req.Gid
 		file.dirtyMetadata = true
 	}
@@ -164,7 +164,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		file.dirtyMetadata = true
 	}
 
-	if req.Valid.Mtime() {
+	if req.Valid.Mtime() && entry.Attributes.Mtime != req.Mtime.Unix() {
 		entry.Attributes.Mtime = req.Mtime.Unix()
 		file.dirtyMetadata = true
 	}
@@ -360,3 +360,30 @@ func (file *File) saveEntry(entry *filer_pb.Entry) error {
 func (file *File) getEntry() *filer_pb.Entry {
 	return file.entry
 }
+
+func (file *File) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entry, error) {
+	err := file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.DownloadToLocalRequest{
+			Directory: file.dir.FullPath(),
+			Name:      entry.Name,
+		}
+
+		glog.V(4).Infof("download entry: %v", request)
+		resp, err := client.DownloadToLocal(context.Background(), request)
+		if err != nil {
+			glog.Errorf("DownloadToLocal file %s/%s: %v", file.dir.FullPath(), file.Name, err)
+			return fuse.EIO
+		}
+
+		entry = resp.Entry
+
+		file.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, resp.Entry))
+
+		file.dirtyMetadata = false
+
+		return nil
+	})
+
+	return entry, err
+}