skip printouts

better fix https://github.com/chrislusf/seaweedfs/issues/1227

.travis.yml

@@ -1,9 +1,9 @@
 sudo: false
 language: go
 go:
-  - 1.11.x
   - 1.12.x
   - 1.13.x
+  - 1.14.x
 
 before_install:
   - export PATH=/home/travis/gopath/bin:$PATH
@@ -45,4 +45,4 @@ deploy:
   on:
     tags: true
     repo: chrislusf/seaweedfs
-    go: 1.13.x
+    go: 1.14.x

README.md

@@ -81,17 +81,15 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two
 1. to store billions of files!
 2. to serve the files fast!
 
-SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages file volumes, and it lets these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (just one disk read operation).
+SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages file volumes, and it lets these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (O(1), usually just one disk read operation).
+
+SeaweedFS can transparently integrate with the cloud. With hot data on local cluster, and warm data on the cloud with O(1) access time, SeaweedFS can achieve both fast local access time and elastic cloud storage capacity, without any client side changes.
 
 There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
 
 SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). Also, SeaweedFS implements erasure coding with ideas from [f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf)
 
-SeaweedFS can work very well with just the object store. [[Filer]] can then be added later to support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql/Postgres/Redis/Etcd/Cassandra/LevelDB/MemSql/TiDB/CockroachDB.
-
-[Back to TOC](#table-of-contents)
-
-## Features ##
+On top of the object store, optional [Filer] can support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql, Postgres, Redis, Etcd, Cassandra, LevelDB, MemSql, TiDB, TiKV, CockroachDB, etc.
 
 [Back to TOC](#table-of-contents)
 
@@ -104,8 +102,10 @@ SeaweedFS can work very well with just the object store. [[Filer]] can then be a
 * Adding/Removing servers does **not** cause any data re-balancing.
 * Optionally fix the orientation for jpeg pictures.
 * Support ETag, Accept-Range, Last-Modified, etc.
-* Support in-memory/leveldb/boltdb/btree mode tuning for memory/performance balance.
+* Support in-memory/leveldb/readonly mode tuning for memory/performance balance.
 * Support rebalancing the writable and readonly volumes.
+* [Transparent cloud integration][CloudTier]: unlimited capacity via tiered cloud storage for warm data.
+* [Erasure Coding for warm storage][ErasureCoding]  Rack-Aware 10.4 erasure coding reduces storage cost and increases availability.
 
 [Back to TOC](#table-of-contents)
 
@@ -113,7 +113,6 @@ SeaweedFS can work very well with just the object store. [[Filer]] can then be a
 * [filer server][Filer] provide "normal" directories and files via http.
 * [mount filer][Mount] to read and write files directly as a local directory via FUSE.
 * [Amazon S3 compatible API][AmazonS3API] to access files with S3 tooling.
-* [Erasure Coding for warm storage][ErasureCoding]  Rack-Aware 10.4 erasure coding reduces storage cost and increases availability.
 * [Hadoop Compatible File System][Hadoop] to access files from Hadoop/Spark/Flink/etc jobs.
 * [Async Backup To Cloud][BackupToCloud] has extremely fast local access and backups to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
 * [WebDAV] access as a mapped drive on Mac and Windows, or from mobile devices.
@@ -125,6 +124,7 @@ SeaweedFS can work very well with just the object store. [[Filer]] can then be a
 [Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
 [WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV
 [ErasureCoding]: https://github.com/chrislusf/seaweedfs/wiki/Erasure-coding-for-warm-storage
+[CloudTier]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Tier
 
 [Back to TOC](#table-of-contents)
 
@@ -318,6 +318,16 @@ Each individual file size is limited to the volume size.
 
 All file meta information stored on an volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
 
+### Tiered Storage to the cloud ###
+
+The local volume servers are much faster, while cloud storages have elastic capacity and are actually more cost-efficient if not accessed often (usually free to upload, but relatively costly to access). With the append-only structure and O(1) access time, SeaweedFS can take advantage of both local and cloud storage by offloading the warm data to the cloud.
+
+Usually hot data are fresh and warm data are old. SeaweedFS puts the newly created volumes on local servers, and optionally upload the older volumes on the cloud. If the older data are accessed less often, this literally gives you unlimited capacity with limited local servers, and still fast for new data. 
+
+With the O(1) access time, the network latency cost is kept at minimum. 
+
+If the hot~warm data is split as 20~80, with 20 servers, you can achieve storage capacity of 100 servers. That's a cost saving of 80%! Or you can repurpose the 80 servers to store new data also, and get 5X storage throughput.
+
 [Back to TOC](#table-of-contents)
 
 ## Compared to Other File Systems ##
@@ -451,50 +461,49 @@ My Own Unscientific Single Machine Results on Mac Book with Solid State Disk, CP
 Write 1 million 1KB file:
 ```
 Concurrency Level:      16
-Time taken for tests:   88.796 seconds
+Time taken for tests:   66.753 seconds
 Complete requests:      1048576
 Failed requests:        0
-Total transferred:      1106764659 bytes
-Requests per second:    11808.87 [#/sec]
-Transfer rate:          12172.05 [Kbytes/sec]
+Total transferred:      1106789009 bytes
+Requests per second:    15708.23 [#/sec]
+Transfer rate:          16191.69 [Kbytes/sec]
 
 Connection Times (ms)
               min      avg        max      std
-Total:        0.2      1.3       44.8      0.9
+Total:        0.3      1.0       84.3      0.9
 
 Percentage of the requests served within a certain time (ms)
-   50%      1.1 ms
-   66%      1.3 ms
-   75%      1.5 ms
-   80%      1.7 ms
-   90%      2.1 ms
-   95%      2.6 ms
-   98%      3.7 ms
-   99%      4.6 ms
-  100%     44.8 ms
+   50%      0.8 ms
+   66%      1.0 ms
+   75%      1.1 ms
+   80%      1.2 ms
+   90%      1.4 ms
+   95%      1.7 ms
+   98%      2.1 ms
+   99%      2.6 ms
+  100%     84.3 ms
 ```
 
 Randomly read 1 million files:
 ```
 Concurrency Level:      16
-Time taken for tests:   34.263 seconds
+Time taken for tests:   22.301 seconds
 Complete requests:      1048576
 Failed requests:        0
-Total transferred:      1106762945 bytes
-Requests per second:    30603.34 [#/sec]
-Transfer rate:          31544.49 [Kbytes/sec]
+Total transferred:      1106812873 bytes
+Requests per second:    47019.38 [#/sec]
+Transfer rate:          48467.57 [Kbytes/sec]
 
 Connection Times (ms)
               min      avg        max      std
-Total:        0.0      0.5       20.7      0.7
+Total:        0.0      0.3       54.1      0.2
 
 Percentage of the requests served within a certain time (ms)
-   50%      0.4 ms
-   75%      0.5 ms
-   95%      0.6 ms
-   98%      0.8 ms
-   99%      1.2 ms
-  100%     20.7 ms
+   50%      0.3 ms
+   90%      0.4 ms
+   98%      0.6 ms
+   99%      0.7 ms
+  100%     54.1 ms
 ```
 
 [Back to TOC](#table-of-contents)
@@ -513,6 +522,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
+The text of this page is available for modification and reuse under the terms of the Creative Commons Attribution-Sharealike 3.0 Unported License and the GNU Free Documentation License (unversioned, with no invariant sections, front-cover texts, or back-cover texts).
+
 [Back to TOC](#table-of-contents)
 
 ## Stargazers over time ##

docker/Dockerfile.go_build

@@ -1,5 +1,15 @@
-FROM golang:latest
-RUN go get github.com/chrislusf/seaweedfs/weed
+FROM frolvlad/alpine-glibc as builder
+RUN apk add git go g++
+RUN mkdir -p /go/src/github.com/chrislusf/
+RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
+RUN cd /go/src/github.com/chrislusf/seaweedfs/weed && go install
+
+FROM alpine AS final
+LABEL author="Chris Lu"
+COPY --from=builder /root/go/bin/weed /usr/bin/
+RUN mkdir -p /etc/seaweedfs
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
 
 # volume server gprc port
 EXPOSE 18080
@@ -20,10 +30,6 @@ RUN mkdir -p /data/filerldb2
 
 VOLUME /data
 
-RUN mkdir -p /etc/seaweedfs
-RUN cp /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
-RUN cp /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
-RUN cp /go/bin/weed /usr/bin/
 
 ENTRYPOINT ["/entrypoint.sh"]

docker/Dockerfile.local

@@ -0,0 +1,29 @@
+FROM alpine AS final
+LABEL author="Chris Lu"
+COPY  ./weed /usr/bin/
+RUN mkdir -p /etc/seaweedfs
+COPY ./filer.toml /etc/seaweedfs/filer.toml
+COPY ./entrypoint.sh /entrypoint.sh
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+
+RUN mkdir -p /data/filerldb2
+
+VOLUME /data
+
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/Makefile

@@ -0,0 +1,19 @@
+all: gen
+
+.PHONY : gen
+
+gen: dev
+
+build:
+	cd ../weed; GOOS=linux go build; mv weed ../docker/
+	docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
+	rm ./weed
+
+dev: build
+	docker-compose -f local-dev-compose.yml -p seaweedfs up
+
+cluster: build
+	docker-compose -f local-cluster-compose.yml -p seaweedfs up
+
+clean:
+	rm ./weed

docker/README.md

@@ -11,11 +11,19 @@ docker-compose -f seaweedfs-compose.yml -p seaweedfs up
 
 ```
 
-## Development
+## Try latest tip
+
+```bash
+
+wget https://raw.githubusercontent.com/chrislusf/seaweedfs/master/docker/seaweedfs-dev-compose.yml
+
+docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
+
+```
+
+## Local Development
 
 ```bash
 cd $GOPATH/src/github.com/chrislusf/seaweedfs/docker
-
-docker-compose -f dev-compose.yml -p seaweedfs up
-
+make
 ```

docker/dev-compose.yml

@@ -1,43 +0,0 @@
-version: '2'
-
-services:
-  master:
-    build:
-      context: .
-      dockerfile: Dockerfile.go_build
-    ports:
-    - 9333:9333
-    - 19333:19333
-    command: "master -ip=master"
-  volume:
-    build:
-      context: .
-      dockerfile: Dockerfile.go_build
-    ports:
-    - 8080:8080
-    - 18080:18080
-    command: 'volume -max=5 -mserver="master:9333" -port=8080'
-    depends_on:
-    - master
-  filer:
-    build:
-      context: .
-      dockerfile: Dockerfile.go_build
-    ports:
-    - 8888:8888
-    - 18888:18888
-    command: 'filer -master="master:9333"'
-    depends_on:
-    - master
-    - volume
-  s3:
-    build:
-      context: .
-      dockerfile: Dockerfile.go_build
-    ports:
-    - 8333:8333
-    command: 's3 -filer="filer:8888"'
-    depends_on:
-    - master
-    - volume
-    - filer

docker/local-cluster-compose.yml

@@ -0,0 +1,53 @@
+version: '2'
+
+services:
+  master0:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335"
+  master1:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9334:9334
+      - 19334:19334
+    command: "master -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335"
+  master2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9335:9335
+      - 19335:19335
+    command: "master -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: '-v=2 volume -max=5 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume'
+    depends_on:
+      - master0
+      - master1
+      - master2
+  filer:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+    command: '-v=4 filer -master="master0:9333,master1:9334,master2:9335"'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8333:8333
+    command: '-v=4 s3 -filer="filer:8888"'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume
+      - filer

docker/local-dev-compose.yml

@@ -0,0 +1,35 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: '-v=2 volume -max=5 -mserver="master:9333" -port=8080 -ip=volume'
+    depends_on:
+      - master
+  filer:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+    command: '-v=4 filer -master="master:9333"'
+    depends_on:
+      - master
+      - volume
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8333:8333
+    command: '-v=4 s3 -filer="filer:8888"'
+    depends_on:
+      - master
+      - volume
+      - filer

docker/seaweedfs-compose.yml

@@ -4,28 +4,28 @@ services:
   master:
     image: chrislusf/seaweedfs # use a remote image
     ports:
-    - 9333:9333
-    - 19333:19333
+      - 9333:9333
+      - 19333:19333
     command: "master -ip=master"
   volume:
     image: chrislusf/seaweedfs # use a remote image
     ports:
-    - 8080:8080
-    - 18080:18080
+      - 8080:8080
+      - 18080:18080
     command: 'volume -max=15 -mserver="master:9333" -port=8080'
     depends_on:
-    - master
+      - master
   filer:
     image: chrislusf/seaweedfs # use a remote image
     ports:
-    - 8888:8888
-    - 18888:18888
+      - 8888:8888
+      - 18888:18888
     command: 'filer -master="master:9333"'
     tty: true
     stdin_open: true
     depends_on:
-    - master
-    - volume
+      - master
+      - volume
   cronjob:
     image: chrislusf/seaweedfs # use a remote image
     command: 'cronjob'
@@ -34,14 +34,14 @@ services:
       CRON_SCHEDULE: '*/2 * * * * *' # Default: '*/5 * * * * *'
       WEED_MASTER: master:9333 # Default: localhost:9333
     depends_on:
-    - master
-    - volume
+      - master
+      - volume
   s3:
     image: chrislusf/seaweedfs # use a remote image
     ports:
-    - 8333:8333
+      - 8333:8333
     command: 's3 -filer="filer:8888"'
     depends_on:
-    - master
-    - volume
-    - filer
+      - master
+      - volume
+      - filer

docker/seaweedfs-dev-compose.yml

@@ -0,0 +1,35 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs:dev # use a remote dev image
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master"
+  volume:
+    image: chrislusf/seaweedfs:dev # use a remote dev image
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: '-v=2 volume -max=5 -mserver="master:9333" -port=8080 -ip=volume'
+    depends_on:
+      - master
+  filer:
+    image: chrislusf/seaweedfs:dev # use a remote dev image
+    ports:
+      - 8888:8888
+      - 18888:18888
+    command: '-v=4 filer -master="master:9333"'
+    depends_on:
+      - master
+      - volume
+  s3:
+    image: chrislusf/seaweedfs:dev # use a remote dev image
+    ports:
+      - 8333:8333
+    command: '-v=4 s3 -filer="filer:8888"'
+    depends_on:
+      - master
+      - volume
+      - filer

go.mod

@@ -69,7 +69,6 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/viper v1.4.0
 	github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 // indirect
-	github.com/stretchr/testify v1.4.0 // indirect
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/tidwall/gjson v1.3.2
 	github.com/tidwall/match v1.0.1

k8s/README.md

@@ -0,0 +1,23 @@
+## SEAWEEDFS - helm chart (2.x)
+
+### info:
+* master/filer/volume are stateful sets with anti-affinity on the hostname,
+so your deployment will be spread/HA.
+* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
+and backup/HA memsql can provide.
+* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer
+with ENV.
+* cert config exists and can be enabled, but not been tested.
+
+### current instances config (AIO):
+1 instance for each type (master/filer/volume/s3)
+
+instances need node labels:
+* sw-volume: true  (for volume instance, specific tag)
+* sw-backend: true (for all others, as they less resource demanding)
+
+you can update the replicas count for each node type in values.yaml,
+need to add more nodes with the corresponding label.
+
+most of the configuration are available through values.yaml
+

k8s/seaweedfs/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

k8s/seaweedfs/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+description: SeaweedFS
+name: seaweedfs
+version: 1.61

k8s/seaweedfs/templates/_helpers.tpl

@@ -0,0 +1,114 @@
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to
+this (by the DNS naming spec). If release name contains chart name it will
+be used as a full name.
+*/}}
+{{- define "seaweedfs.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "seaweedfs.chart" -}}
+{{- printf "%s-helm" .Chart.Name | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "seaweedfs.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Inject extra environment vars in the format key:value, if populated
+*/}}
+{{- define "seaweedfs.extraEnvironmentVars" -}}
+{{- if .extraEnvironmentVars -}}
+{{- range $key, $value := .extraEnvironmentVars }}
+- name: {{ $key }}
+  value: {{ $value | quote }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Return the proper filer image */}}
+{{- define "filer.image" -}}
+{{- if .Values.filer.imageOverride -}}
+{{- $imageOverride := .Values.filer.imageOverride -}}
+{{- printf "%s" $imageOverride -}}
+{{- else -}}
+{{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}}
+{{- $repositoryName := .Values.image.repository | toString -}}
+{{- $name := .Values.global.imageName | toString -}}
+{{- $tag := .Values.global.imageTag | toString -}}
+{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Return the proper postgresqlSchema image */}}
+{{- define "filer.dbSchema.image" -}}
+{{- if .Values.filer.dbSchema.imageOverride -}}
+{{- $imageOverride := .Values.filer.dbSchema.imageOverride -}}
+{{- printf "%s" $imageOverride -}}
+{{- else -}}
+{{- $registryName := default .Values.global.registry .Values.global.localRegistry | toString -}}
+{{- $repositoryName := .Values.global.repository | toString -}}
+{{- $name := .Values.filer.dbSchema.imageName | toString -}}
+{{- $tag := .Values.filer.dbSchema.imageTag | toString -}}
+{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Return the proper master image */}}
+{{- define "master.image" -}}
+{{- if .Values.master.imageOverride -}}
+{{- $imageOverride := .Values.master.imageOverride -}}
+{{- printf "%s" $imageOverride -}}
+{{- else -}}
+{{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}}
+{{- $repositoryName := .Values.image.repository | toString -}}
+{{- $name := .Values.global.imageName | toString -}}
+{{- $tag := .Values.global.imageTag | toString -}}
+{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Return the proper s3 image */}}
+{{- define "s3.image" -}}
+{{- if .Values.s3.imageOverride -}}
+{{- $imageOverride := .Values.s3.imageOverride -}}
+{{- printf "%s" $imageOverride -}}
+{{- else -}}
+{{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}}
+{{- $repositoryName := .Values.image.repository | toString -}}
+{{- $name := .Values.global.imageName | toString -}}
+{{- $tag := .Values.global.imageTag | toString -}}
+{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Return the proper volume image */}}
+{{- define "volume.image" -}}
+{{- if .Values.volume.imageOverride -}}
+{{- $imageOverride := .Values.volume.imageOverride -}}
+{{- printf "%s" $imageOverride -}}
+{{- else -}}
+{{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}}
+{{- $repositoryName := .Values.image.repository | toString -}}
+{{- $name := .Values.global.imageName | toString -}}
+{{- $tag := .Values.global.imageTag | toString -}}
+{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
+{{- end -}}
+{{- end -}}

k8s/seaweedfs/templates/ca-cert.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.global.enableSecurity }}
+apiVersion: certmanager.k8s.io/v1alpha1
+kind: Certificate
+metadata:
+  name: {{ template "seaweedfs.name" . }}-ca-cert
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretName: {{ template "seaweedfs.name" . }}-ca-cert
+  commonName: "{{ template "seaweedfs.name" . }}-root-ca"
+  isCA: true
+  issuerRef:
+    name: {{ template "seaweedfs.name" . }}-clusterissuer
+    kind: ClusterIssuer
+{{- end }}

k8s/seaweedfs/templates/cert-clusterissuer.yaml

@@ -0,0 +1,8 @@
+{{- if .Values.global.enableSecurity }}
+apiVersion: certmanager.k8s.io/v1alpha1
+kind: ClusterIssuer
+metadata:
+  name: {{ template "seaweedfs.name" . }}-clusterissuer
+spec:
+  selfSigned: {}
+{{- end }}

k8s/seaweedfs/templates/client-cert.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.global.enableSecurity }}
+apiVersion: certmanager.k8s.io/v1alpha1
+kind: Certificate
+metadata:
+  name: {{ template "seaweedfs.name" . }}-client-cert
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretName: {{ template "seaweedfs.name" . }}-client-cert
+  issuerRef:
+    name: {{ template "seaweedfs.name" . }}-clusterissuer
+    kind: ClusterIssuer
+  commonName: {{ .Values.certificates.commonName }}
+  organization:
+    - "SeaweedFS CA"
+  dnsNames:
+    - '*.{{ .Release.Namespace }}'
+    - '*.{{ .Release.Namespace }}.svc'
+    - '*.{{ .Release.Namespace }}.svc.cluster.local'
+    - '*.{{ template "seaweedfs.name" . }}-master'
+    - '*.{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}'
+    - '*.{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}.svc'
+    - '*.{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}.svc.cluster.local'
+{{- if .Values.certificates.ipAddresses }}
+  ipAddresses:
+    {{- range .Values.certificates.ipAddresses }}
+    - {{ . }}
+    {{- end }}
+{{- end }}
+  keyAlgorithm: {{ .Values.certificates.keyAlgorithm }}
+  keySize: {{ .Values.certificates.keySize }}
+  duration: {{ .Values.certificates.duration }}
+  renewBefore: {{ .Values.certificates.renewBefore }}
+{{- end }}

k8s/seaweedfs/templates/filer-cert.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.global.enableSecurity }}
+apiVersion: certmanager.k8s.io/v1alpha1
+kind: Certificate
+metadata:
+  name: {{ template "seaweedfs.name" . }}-filer-cert
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretName: {{ template "seaweedfs.name" . }}-filer-cert
+  issuerRef:
+    name: {{ template "seaweedfs.name" . }}-clusterissuer
+    kind: ClusterIssuer
+  commonName: {{ .Values.certificates.commonName }}
+  organization:
+    - "SeaweedFS CA"
+  dnsNames:
+    - '*.{{ .Release.Namespace }}'
+    - '*.{{ .Release.Namespace }}.svc'
+    - '*.{{ .Release.Namespace }}.svc.cluster.local'
+    - '*.{{ template "seaweedfs.name" . }}-master'
+    - '*.{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}'
+    - '*.{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}.svc'
+    - '*.{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}.svc.cluster.local'
+{{- if .Values.certificates.ipAddresses }}
+  ipAddresses:
+    {{- range .Values.certificates.ipAddresses }}
+    - {{ . }}
+    {{- end }}
+{{- end }}
+  keyAlgorithm: {{ .Values.certificates.keyAlgorithm }}
+  keySize: {{ .Values.certificates.keySize }}
+  duration: {{ .Values.certificates.duration }}
+  renewBefore: {{ .Values.certificates.renewBefore }}
+{{- end }}

k8s/seaweedfs/templates/filer-service.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "seaweedfs.name" . }}-filer
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    component: filer
+spec:
+  clusterIP: None
+  ports:
+  - name: "swfs-filer"
+    port: {{ .Values.filer.port }}
+    targetPort: {{ .Values.filer.port }}
+    protocol: TCP
+  - name: "swfs-filer-grpc"
+    port: {{ .Values.filer.grpcPort }}
+    targetPort: {{ .Values.filer.grpcPort }}
+    protocol: TCP
+  selector:
+    app: {{ template "seaweedfs.name" . }}
+    component: filer

k8s/seaweedfs/templates/filer-statefulset.yaml

@@ -0,0 +1,207 @@
+{{- if .Values.filer.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ template "seaweedfs.name" . }}-filer
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    chart: {{ template "seaweedfs.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+spec:
+  serviceName: {{ template "seaweedfs.name" . }}-filer
+  podManagementPolicy: Parallel
+  replicas: {{ .Values.filer.replicas }}
+  {{- if (gt (int .Values.filer.updatePartition) 0) }}
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      partition: {{ .Values.filer.updatePartition }}
+  {{- end }}
+  selector:
+    matchLabels:
+      app: {{ template "seaweedfs.name" . }}
+      chart: {{ template "seaweedfs.chart" . }}
+      release: {{ .Release.Name }}
+      component: filer
+  template:
+    metadata:
+      labels:
+        app: {{ template "seaweedfs.name" . }}
+        chart: {{ template "seaweedfs.chart" . }}
+        release: {{ .Release.Name }}
+        component: filer
+    spec:
+      restartPolicy: {{ default .Values.global.restartPolicy .Values.filer.restartPolicy }}
+      {{- if .Values.filer.affinity }}
+      affinity:
+        {{ tpl .Values.filer.affinity . | nindent 8 | trim }}
+      {{- end }}
+      {{- if .Values.filer.tolerations }}
+      tolerations:
+        {{ tpl .Values.filer.tolerations . | nindent 8 | trim }}
+      {{- end }}
+      {{- if .Values.global.imagePullSecrets }}
+      imagePullSecrets:
+        - name: {{ .Values.global.imagePullSecrets }}
+      {{- end }}
+      serviceAccountName: seaweefds-rw-sa #hack for delete pod master after migration
+      terminationGracePeriodSeconds: 60
+      {{- if .Values.filer.priorityClassName }}
+      priorityClassName: {{ .Values.filer.priorityClassName | quote }}
+      {{- end }}
+      enableServiceLinks: false
+      containers:
+        - name: seaweedfs
+          image: {{ template "filer.image" . }}
+          imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: WEED_MYSQL_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: secret-seaweedfs-db
+                  key: user
+            - name: WEED_MYSQL_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: secret-seaweedfs-db
+                  key: password
+            - name: SEAWEEDFS_FULLNAME
+              value: "{{ template "seaweedfs.name" . }}"
+            {{- if .Values.filer.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.filer.extraEnvironmentVars }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+            {{- end }}
+            {{- end }}
+          command:
+            - "/bin/sh"
+            - "-ec"
+            - | 
+              exec /usr/bin/weed -logdir=/logs \
+              {{- if .Values.filer.loggingOverrideLevel }}
+              -v={{ .Values.filer.loggingOverrideLevel }} \
+              {{- else }}
+              -v={{ .Values.global.loggingLevel }} \
+              {{- end }}
+              filer \
+              -port={{ .Values.filer.port }} \
+              {{- if .Values.filer.disableHttp }}
+              -disableHttp \
+              {{- end }}
+              {{- if .Values.filer.disableDirListing }}
+              -disableDirListing \
+              {{- end }}
+              -dirListLimit={{ .Values.filer.dirListLimit }} \
+              -ip=${POD_IP} \
+              -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
+          {{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }}    
+          volumeMounts:
+            - name: seaweedfs-filer-log-volume
+              mountPath: "/logs/"
+            {{- if .Values.global.enableSecurity }}
+            - name: security-config
+              readOnly: true
+              mountPath: /etc/seaweedfs/security.toml
+              subPath: security.toml
+            - name: ca-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/ca/
+            - name: master-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/master/
+            - name: volume-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/volume/
+            - name: filer-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/filer/
+            - name: client-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/client/
+            {{- end }}
+            {{ tpl .Values.filer.extraVolumeMounts . | nindent 12 | trim }}
+          {{- end }}
+          ports:
+            - containerPort: {{ .Values.filer.port }}
+              name: swfs-filer
+            - containerPort: {{ .Values.filer.grpcPort }}
+              #name: swfs-filer-grpc
+          readinessProbe:
+            httpGet:
+              path: /
+              port: {{ .Values.filer.port }}
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 15
+            successThreshold: 1
+            failureThreshold: 100
+          livenessProbe:
+            httpGet:
+              path: /
+              port: {{ .Values.filer.port }}
+              scheme: HTTP
+            initialDelaySeconds: 20
+            periodSeconds: 30
+            successThreshold: 1
+            failureThreshold: 5
+          {{- if .Values.filer.resources }}
+          resources:
+            {{ tpl .Values.filer.resources . | nindent 12 | trim }}
+          {{- end }}
+      volumes:
+        - name: seaweedfs-filer-log-volume
+          hostPath:
+            path: /storage/logs/seaweedfs/filer
+            type: DirectoryOrCreate
+        {{- if .Values.global.enableSecurity }}
+        - name: security-config
+          configMap:
+            name: {{ template "seaweedfs.name" . }}-security-config
+        - name: ca-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-ca-cert
+        - name: master-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-master-cert
+        - name: volume-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-volume-cert
+        - name: filer-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-filer-cert
+        - name: client-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-client-cert
+        {{- end }}
+        {{ tpl .Values.filer.extraVolumes . | indent 8 | trim }}
+      {{- if .Values.filer.nodeSelector }}
+      nodeSelector:
+        {{ tpl .Values.filer.nodeSelector . | indent 8 | trim }}
+      {{- end }}
+{{/*  volumeClaimTemplates:*/}}
+{{/*    - metadata:*/}}
+{{/*        name: data-{{ .Release.Namespace }}*/}}
+{{/*      spec:*/}}
+{{/*        accessModes:*/}}
+{{/*          - ReadWriteOnce*/}}
+{{/*        resources:*/}}
+{{/*          requests:*/}}
+{{/*            storage: {{ .Values.filer.storage }}*/}}
+{{/*        {{- if .Values.filer.storageClass }}*/}}
+{{/*        storageClassName: {{ .Values.filer.storageClass }}*/}}
+{{/*      {{- end }}*/}}
+{{- end }}

k8s/seaweedfs/templates/ingress.yaml

@@ -0,0 +1,59 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+ name: ingress-{{ template "seaweedfs.name" . }}-filer
+ annotations:
+   kubernetes.io/ingress.class: "nginx"
+   nginx.ingress.kubernetes.io/auth-type: "basic"
+   nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
+   nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Filer'
+   nginx.ingress.kubernetes.io/service-upstream: "true"
+   nginx.ingress.kubernetes.io/rewrite-target: /$1
+   nginx.ingress.kubernetes.io/use-regex: "true"
+   nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
+   nginx.ingress.kubernetes.io/ssl-redirect: "false"
+   nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+   nginx.ingress.kubernetes.io/configuration-snippet: |
+     sub_filter '<head>' '<head> <base href="/sw-filer/">'; #add base url
+     sub_filter '="/' '="./';                               #make absolute paths to relative
+     sub_filter '=/' '=./';
+     sub_filter '/seaweedfsstatic' './seaweedfsstatic';
+     sub_filter_once off;
+spec:
+ rules:
+ - http:
+     paths:
+     - path: /sw-filer/?(.*)
+       backend:
+         serviceName: {{ template "seaweedfs.name" . }}-filer
+         servicePort: {{ .Values.filer.port }}
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: ingress-{{ template "seaweedfs.name" . }}-master
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/auth-type: "basic"
+    nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
+    nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master'
+    nginx.ingress.kubernetes.io/service-upstream: "true"
+    nginx.ingress.kubernetes.io/rewrite-target: /$1
+    nginx.ingress.kubernetes.io/use-regex: "true"
+    nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
+    nginx.ingress.kubernetes.io/ssl-redirect: "false"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+    nginx.ingress.kubernetes.io/configuration-snippet: |
+      sub_filter '<head>' '<head> <base href="/sw-master/">'; #add base url
+      sub_filter '="/' '="./';                                #make absolute paths to relative
+      sub_filter '=/' '=./';
+      sub_filter '/seaweedfsstatic' './seaweedfsstatic';
+      sub_filter_once off;
+spec:
+  rules:
+  - http:
+      paths:
+      - path: /sw-master/?(.*)
+        backend:
+          serviceName: {{ template "seaweedfs.name" . }}-master
+          servicePort: {{ .Values.master.port }}

k8s/seaweedfs/templates/master-cert.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.global.enableSecurity }}
+apiVersion: certmanager.k8s.io/v1alpha1
+kind: Certificate
+metadata:
+  name: {{ template "seaweedfs.name" . }}-master-cert
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretName: {{ template "seaweedfs.name" . }}-master-cert
+  issuerRef:
+    name: {{ template "seaweedfs.name" . }}-clusterissuer
+    kind: ClusterIssuer
+  commonName: {{ .Values.certificates.commonName }}
+  organization:
+    - "SeaweedFS CA"
+  dnsNames:
+    - '*.{{ .Release.Namespace }}'
+    - '*.{{ .Release.Namespace }}.svc'
+    - '*.{{ .Release.Namespace }}.svc.cluster.local'
+    - '*.{{ template "seaweedfs.name" . }}-master'
+    - '*.{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}'
+    - '*.{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}.svc'
+    - '*.{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}.svc.cluster.local'
+{{- if .Values.certificates.ipAddresses }}
+  ipAddresses:
+    {{- range .Values.certificates.ipAddresses }}
+    - {{ . }}
+    {{- end }}
+{{- end }}
+  keyAlgorithm: {{ .Values.certificates.keyAlgorithm }}
+  keySize: {{ .Values.certificates.keySize }}
+  duration: {{ .Values.certificates.duration }}
+  renewBefore: {{ .Values.certificates.renewBefore }}
+{{- end }}

k8s/seaweedfs/templates/master-service.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "seaweedfs.name" . }}-master
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    component: master
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"    
+spec:
+  clusterIP: None
+  ports:
+  - name: "swfs-master"
+    port: {{ .Values.master.port }}
+    targetPort: {{ .Values.master.port }}
+    protocol: TCP
+  - name: "swfs-master-grpc"
+    port: {{ .Values.master.grpcPort }}
+    targetPort: {{ .Values.master.grpcPort }}
+    protocol: TCP
+  selector:
+    app: {{ template "seaweedfs.name" . }}
+    component: master

k8s/seaweedfs/templates/master-statefulset.yaml

@@ -0,0 +1,199 @@
+{{- if .Values.master.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ template "seaweedfs.name" . }}-master
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    chart: {{ template "seaweedfs.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+spec:
+  serviceName: {{ template "seaweedfs.name" . }}-master
+  podManagementPolicy: Parallel
+  replicas: {{ .Values.master.replicas }}
+  {{- if (gt (int .Values.master.updatePartition) 0) }}
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      partition: {{ .Values.master.updatePartition }}
+  {{- end }}
+  selector:
+    matchLabels:
+      app: {{ template "seaweedfs.name" . }}
+      chart: {{ template "seaweedfs.chart" . }}
+      release: {{ .Release.Name }}
+      component: master
+  template:
+    metadata:
+      labels:
+        app: {{ template "seaweedfs.name" . }}
+        chart: {{ template "seaweedfs.chart" . }}
+        release: {{ .Release.Name }}
+        component: master
+    spec:
+      restartPolicy: {{ default .Values.global.restartPolicy .Values.master.restartPolicy }}
+      {{- if .Values.master.affinity }}
+      affinity:
+        {{ tpl .Values.master.affinity . | nindent 8 | trim }}
+      {{- end }}
+      {{- if .Values.master.tolerations }}
+      tolerations:
+        {{ tpl .Values.master.tolerations . | nindent 8 | trim }}
+      {{- end }}
+      {{- if .Values.global.imagePullSecrets }}
+      imagePullSecrets:
+        - name: {{ .Values.global.imagePullSecrets }}
+      {{- end }}
+      terminationGracePeriodSeconds: 60
+      {{- if .Values.master.priorityClassName }}
+      priorityClassName: {{ .Values.master.priorityClassName | quote }}
+      {{- end }}
+      enableServiceLinks: false
+      containers:
+        - name: seaweedfs
+          image: {{ template "master.image" . }}
+          imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: SEAWEEDFS_FULLNAME
+              value: "{{ template "seaweedfs.name" . }}"
+          command:
+            - "/bin/sh"
+            - "-ec"
+            - | 
+              exec /usr/bin/weed -logdir=/logs \
+              {{- if .Values.master.loggingOverrideLevel }}
+              -v={{ .Values.master.loggingOverrideLevel }} \
+              {{- else }}
+              -v={{ .Values.global.loggingLevel }} \
+              {{- end }}
+              master \
+              -port={{ .Values.master.port }} \
+              -mdir=/data \
+              -ip.bind={{ .Values.master.ipBind }} \
+              {{- if .Values.master.volumePreallocate }}
+              -volumePreallocate \
+              {{- end }}
+              {{- if .Values.global.monitoring.enabled }}
+              -metrics.address="{{ .Values.global.monitoring.gatewayHost }}:{{ .Values.global.monitoring.gatewayPort }}" \
+              {{- end }}
+              -volumeSizeLimitMB={{ .Values.master.volumeSizeLimitMB }} \
+              {{- if .Values.master.disableHttp }}
+              -disableHttp \
+              {{- end }}
+              -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master \
+              -peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
+          volumeMounts:
+            - name : data-{{ .Release.Namespace }}
+              mountPath: /data
+            - name: seaweedfs-master-log-volume
+              mountPath: "/logs/"
+            {{- if .Values.global.enableSecurity }}
+            - name: security-config
+              readOnly: true
+              mountPath: /etc/seaweedfs/security.toml
+              subPath: security.toml
+            - name: ca-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/ca/
+            - name: master-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/master/
+            - name: volume-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/volume/
+            - name: filer-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/filer/
+            - name: client-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/client/
+            {{- end }}
+            {{ tpl .Values.master.extraVolumeMounts . | nindent 12 | trim }}
+          ports:
+            - containerPort: {{ .Values.master.port }}
+              name: swfs-master
+            - containerPort: {{ .Values.master.grpcPort }}
+              #name: swfs-master-grpc
+          readinessProbe:
+            httpGet:
+              path: /cluster/status
+              port: {{ .Values.master.port }}
+              scheme: HTTP
+            initialDelaySeconds: 5
+            periodSeconds: 15
+            successThreshold: 2
+            failureThreshold: 100
+          livenessProbe:
+            httpGet:
+              path: /cluster/status
+              port: {{ .Values.master.port }}
+              scheme: HTTP
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 6
+          {{- if .Values.master.resources }}
+          resources:
+            {{ tpl .Values.master.resources . | nindent 12 | trim }}
+          {{- end }}
+      volumes:
+        - name: seaweedfs-master-log-volume
+          hostPath:
+            path: /storage/logs/seaweedfs/master
+            type: DirectoryOrCreate
+        - name: data-{{ .Release.Namespace }}
+          hostPath:
+            path: /ssd/seaweed-master/
+            type: DirectoryOrCreate
+        {{- if .Values.global.enableSecurity }}
+        - name: security-config
+          configMap:
+            name: {{ template "seaweedfs.name" . }}-security-config
+        - name: ca-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-ca-cert
+        - name: master-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-master-cert
+        - name: volume-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-volume-cert
+        - name: filer-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-filer-cert
+        - name: client-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-client-cert
+        {{- end }}
+        {{ tpl .Values.master.extraVolumes . | indent 8 | trim }}
+      {{- if .Values.master.nodeSelector }}
+      nodeSelector:
+        {{ tpl .Values.master.nodeSelector . | indent 8 | trim }}
+      {{- end }}
+{{/*  volumeClaimTemplates:*/}}
+{{/*    - metadata:*/}}
+{{/*        name: data-{{ .Release.Namespace }}*/}}
+{{/*      spec:*/}}
+{{/*        accessModes:*/}}
+{{/*          - ReadWriteOnce*/}}
+{{/*        resources:*/}}
+{{/*          requests:*/}}
+{{/*            storage: {{ .Values.master.storage }}*/}}
+{{/*        {{- if .Values.master.storageClass }}*/}}
+{{/*        storageClassName: {{ .Values.master.storageClass }}*/}}
+{{/*        {{- end }}*/}}
+{{- end }}

k8s/seaweedfs/templates/s3-deployment.yaml

@@ -0,0 +1,158 @@
+{{- if .Values.s3.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "seaweedfs.name" . }}-s3
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    chart: {{ template "seaweedfs.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+spec:
+  serviceName: {{ template "seaweedfs.name" . }}-s3
+  replicas: {{ .Values.s3.replicas }}
+  selector:
+    matchLabels:
+      app: {{ template "seaweedfs.name" . }}
+      chart: {{ template "seaweedfs.chart" . }}
+      release: {{ .Release.Name }}
+      component: s3
+  template:
+    metadata:
+      labels:
+        app: {{ template "seaweedfs.name" . }}
+        chart: {{ template "seaweedfs.chart" . }}
+        release: {{ .Release.Name }}
+        component: s3
+    spec:
+      restartPolicy: {{ default .Values.global.restartPolicy .Values.s3.restartPolicy }}
+      {{- if .Values.s3.tolerations }}
+      tolerations:
+        {{ tpl .Values.s3.tolerations . | nindent 8 | trim }}
+      {{- end }}
+      {{- if .Values.global.imagePullSecrets }}
+      imagePullSecrets:
+        - name: {{ .Values.global.imagePullSecrets }}
+      {{- end }}
+      terminationGracePeriodSeconds: 10
+      {{- if .Values.s3.priorityClassName }}
+      priorityClassName: {{ .Values.s3.priorityClassName | quote }}
+      {{- end }}
+      enableServiceLinks: false
+      containers:
+        - name: seaweedfs
+          image: {{ template "s3.image" . }}
+          imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: SEAWEEDFS_FULLNAME
+              value: "{{ template "seaweedfs.name" . }}"
+          command:
+            - "/bin/sh"
+            - "-ec"
+            - | 
+              exec /usr/bin/weed \
+              {{- if .Values.s3.loggingOverrideLevel }}
+              -v={{ .Values.s3.loggingOverrideLevel }} \
+              {{- else }}
+              -v={{ .Values.global.loggingLevel }} \
+              {{- end }}
+              s3 \
+              -port={{ .Values.s3.port }} \
+              {{- if .Values.global.enableSecurity }}
+              -cert.file=/usr/local/share/ca-certificates/client/tls.crt \
+              -key.file=/usr/local/share/ca-certificates/client/tls.key \
+              {{- end }}
+              {{- if .Values.s3.domainName }}
+              -domainName={{ .Values.s3.domainName }} \
+              {{- end }}
+              -filer={{ template "seaweedfs.name" . }}-filer:{{ .Values.filer.port }}
+          {{- if or (.Values.global.enableSecurity) (.Values.s3.extraVolumeMounts) }}
+          volumeMounts:
+            {{- if .Values.global.enableSecurity }}
+            - name: security-config
+              readOnly: true
+              mountPath: /etc/seaweedfs/security.toml
+              subPath: security.toml
+            - name: ca-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/ca/
+            - name: master-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/master/
+            - name: volume-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/volume/
+            - name: filer-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/filer/
+            - name: client-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/client/
+            {{- end }}
+            {{ tpl .Values.s3.extraVolumeMounts . | nindent 12 | trim }}
+          {{- end }}
+          ports:
+            - containerPort: {{ .Values.s3.port }}
+              name: swfs-s3
+          readinessProbe:
+            httpGet:
+              path: /
+              port: {{ .Values.s3.port }}
+              scheme: HTTP
+            initialDelaySeconds: 15
+            periodSeconds: 15
+            successThreshold: 1
+            failureThreshold: 100
+          livenessProbe:
+            httpGet:
+              path: /
+              port: {{ .Values.s3.port }}
+              scheme: HTTP
+            initialDelaySeconds: 20
+            periodSeconds: 60
+            successThreshold: 1
+            failureThreshold: 20
+          {{- if .Values.s3.resources }}
+          resources:
+            {{ tpl .Values.s3.resources . | nindent 12 | trim }}
+          {{- end }}
+      volumes:
+        {{- if .Values.global.enableSecurity }}
+        - name: security-config
+          configMap:
+            name: {{ template "seaweedfs.name" . }}-security-config
+        - name: ca-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-ca-cert
+        - name: master-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-master-cert
+        - name: volume-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-volume-cert
+        - name: filer-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-filer-cert
+        - name: client-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-client-cert
+        {{- end }}
+        {{ tpl .Values.s3.extraVolumes . | indent 8 | trim }}
+      {{- if .Values.s3.nodeSelector }}
+      nodeSelector:
+        {{ tpl .Values.s3.nodeSelector . | indent 8 | trim }}
+      {{- end }}
+{{- end }}

k8s/seaweedfs/templates/s3-service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "seaweedfs.name" . }}-s3
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    component: s3
+spec:
+  ports:
+  - name: "swfs-s3"
+    port: {{ .Values.s3.port }}
+    targetPort: {{ .Values.s3.port }}
+    protocol: TCP
+  selector:
+    app: {{ template "seaweedfs.name" . }}
+    component: s3

k8s/seaweedfs/templates/secret-seaweedfs-db.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: secret-seaweedfs-db
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/resource-policy": keep
+    "helm.sh/hook": "pre-install"
+stringData:
+  user: "YourSWUser"
+  password: "HardCodedPassword"
+  # better to random generate and create in DB
+  # password: {{ randAlphaNum 10 | sha256sum | b64enc | trunc 32 }}

k8s/seaweedfs/templates/security-configmap.yaml

@@ -0,0 +1,52 @@
+{{- if .Values.global.enableSecurity }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "seaweedfs.name" . }}-security-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    chart: {{ template "seaweedfs.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+data:
+  security.toml: |-
+    # this file is read by master, volume server, and filer
+
+    # the jwt signing key is read by master and volume server
+    # a jwt expires in 10 seconds
+    [jwt.signing]
+    key = "{{ randAlphaNum 10 | b64enc }}"
+
+    # all grpc tls authentications are mutual
+    # the values for the following ca, cert, and key are paths to the PERM files.
+    [grpc]
+    ca = "/usr/local/share/ca-certificates/ca/tls.crt"
+
+    [grpc.volume]
+    cert = "/usr/local/share/ca-certificates/volume/tls.crt"
+    key  = "/usr/local/share/ca-certificates/volume/tls.key"
+
+    [grpc.master]
+    cert = "/usr/local/share/ca-certificates/master/tls.crt"
+    key  = "/usr/local/share/ca-certificates/master/tls.key"
+
+    [grpc.filer]
+    cert = "/usr/local/share/ca-certificates/filer/tls.crt"
+    key  = "/usr/local/share/ca-certificates/filer/tls.key"
+
+    # use this for any place needs a grpc client
+    # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
+    [grpc.client]
+    cert = "/usr/local/share/ca-certificates/client/tls.crt"
+    key  = "/usr/local/share/ca-certificates/client/tls.key"
+
+    # volume server https options
+    # Note: work in progress!
+    #     this does not work with other clients, e.g., "weed filer|mount" etc, yet.
+    [https.client]
+    enabled = false
+    [https.volume]
+    cert = ""
+    key  = ""
+{{- end }}

k8s/seaweedfs/templates/service-account.yaml

@@ -0,0 +1,29 @@
+#hack for delete pod master after migration
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: seaweefds-rw-cr
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: seaweefds-rw-sa
+  namespace: {{ .Release.Namespace }}
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: system:serviceaccount:seaweefds-rw-sa:default
+subjects:
+- kind: ServiceAccount
+  name: seaweefds-rw-sa
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: seaweefds-rw-cr

k8s/seaweedfs/templates/volume-cert.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.global.enableSecurity }}
+apiVersion: certmanager.k8s.io/v1alpha1
+kind: Certificate
+metadata:
+  name: {{ template "seaweedfs.name" . }}-volume-cert
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretName: {{ template "seaweedfs.name" . }}-volume-cert
+  issuerRef:
+    name: {{ template "seaweedfs.name" . }}-clusterissuer
+    kind: ClusterIssuer
+  commonName: {{ .Values.certificates.commonName }}
+  organization:
+    - "SeaweedFS CA"
+  dnsNames:
+    - '*.{{ .Release.Namespace }}'
+    - '*.{{ .Release.Namespace }}.svc'
+    - '*.{{ .Release.Namespace }}.svc.cluster.local'
+    - '*.{{ template "seaweedfs.name" . }}-master'
+    - '*.{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}'
+    - '*.{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}.svc'
+    - '*.{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}.svc.cluster.local'
+{{- if .Values.certificates.ipAddresses }}
+  ipAddresses:
+    {{- range .Values.certificates.ipAddresses }}
+    - {{ . }}
+    {{- end }}
+{{- end }}
+  keyAlgorithm: {{ .Values.certificates.keyAlgorithm }}
+  keySize: {{ .Values.certificates.keySize }}
+  duration: {{ .Values.certificates.duration }}
+  renewBefore: {{ .Values.certificates.renewBefore }}
+{{- end }}

k8s/seaweedfs/templates/volume-service.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "seaweedfs.name" . }}-volume
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    component: volume
+spec:
+  clusterIP: None
+  ports:
+  - name: "swfs-volume"
+    port: {{ .Values.volume.port }}
+    targetPort: {{ .Values.volume.port }}
+    protocol: TCP
+  - name: "swfs-volume-18080"
+    port: {{ .Values.volume.grpcPort }}
+    targetPort: {{ .Values.volume.grpcPort }}
+    protocol: TCP
+  selector:
+    app: {{ template "seaweedfs.name" . }}
+    component: volume

k8s/seaweedfs/templates/volume-statefulset.yaml

@@ -0,0 +1,187 @@
+{{- if .Values.volume.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ template "seaweedfs.name" . }}-volume
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "seaweedfs.name" . }}
+    chart: {{ template "seaweedfs.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+spec:
+  serviceName: {{ template "seaweedfs.name" . }}-volume
+  replicas: {{ .Values.volume.replicas }}
+  selector:
+    matchLabels:
+      app: {{ template "seaweedfs.name" . }}
+      chart: {{ template "seaweedfs.chart" . }}
+      release: {{ .Release.Name }}
+      component: volume
+  template:
+    metadata:
+      labels:
+        app: {{ template "seaweedfs.name" . }}
+        chart: {{ template "seaweedfs.chart" . }}
+        release: {{ .Release.Name }}
+        component: volume
+    spec:
+      {{- if .Values.volume.affinity }}
+      affinity:
+        {{ tpl .Values.volume.affinity . | nindent 8 | trim }}
+      {{- end }}
+      restartPolicy: {{ default .Values.global.restartPolicy .Values.volume.restartPolicy }}
+      {{- if .Values.volume.tolerations }}
+      tolerations:
+        {{ tpl .Values.volume.tolerations . | nindent 8 | trim }}
+      {{- end }}
+      {{- if .Values.global.imagePullSecrets }}
+      imagePullSecrets:
+        - name: {{ .Values.global.imagePullSecrets }}
+      {{- end }}
+      terminationGracePeriodSeconds: 10
+      {{- if .Values.volume.priorityClassName }}
+      priorityClassName: {{ .Values.volume.priorityClassName | quote }}
+      {{- end }}
+      enableServiceLinks: false
+      containers:
+        - name: seaweedfs
+          image: {{ template "volume.image" . }}
+          imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: HOST_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+            - name: SEAWEEDFS_FULLNAME
+              value: "{{ template "seaweedfs.name" . }}"                  
+          command:
+            - "/bin/sh"
+            - "-ec"
+            - |
+              exec /usr/bin/weed -logdir=/logs \
+                {{- if .Values.volume.loggingOverrideLevel }}
+                -v={{ .Values.volume.loggingOverrideLevel }} \
+                {{- else }}
+                -v={{ .Values.global.loggingLevel }} \
+                {{- end }}
+                volume \
+                -port={{ .Values.volume.port }} \
+                -dir={{ .Values.volume.dir }} \
+                -max={{ .Values.volume.maxVolumes }} \
+                {{- if .Values.volume.rack }}
+                -rack={{ .Values.volume.rack }} \
+                {{- end }}
+                {{- if .Values.volume.dataCenter }}
+                -dataCenter={{ .Values.volume.dataCenter }} \
+                {{- end }}
+                -ip.bind={{ .Values.volume.ipBind }} \
+                -read.redirect={{ .Values.volume.readRedirect }} \
+                {{- if .Values.volume.whiteList }}
+                -whiteList={{ .Values.volume.whiteList }} \
+                {{- end }}
+                {{- if .Values.volume.imagesFixOrientation }}
+                -images.fix.orientation \
+                {{- end }}
+                -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume \
+                -compactionMBps={{ .Values.volume.compactionMBps }} \
+                -mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
+          volumeMounts:
+            - name: seaweedfs-volume-storage
+              mountPath: "/data/"
+            - name: seaweedfs-volume-log-volume
+              mountPath: "/logs/"
+            {{- if .Values.global.enableSecurity }}
+            - name: security-config
+              readOnly: true
+              mountPath: /etc/seaweedfs/security.toml
+              subPath: security.toml
+            - name: ca-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/ca/
+            - name: master-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/master/
+            - name: volume-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/volume/
+            - name: filer-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/filer/
+            - name: client-cert
+              readOnly: true
+              mountPath: /usr/local/share/ca-certificates/client/
+            {{- end }}
+            {{ tpl .Values.volume.extraVolumeMounts . | nindent 12 | trim }}
+          ports:
+            - containerPort: {{ .Values.volume.port }}
+              name: swfs-vol
+            - containerPort: {{ .Values.volume.grpcPort }}
+              #name: swfs-vol-grpc
+          readinessProbe:
+            httpGet:
+              path: /status
+              port: {{ .Values.volume.port }}
+              scheme: HTTP
+            initialDelaySeconds: 5
+            periodSeconds: 15
+            successThreshold: 1
+            failureThreshold: 100
+          livenessProbe:
+            httpGet:
+              path: /status
+              port: {{ .Values.volume.port }}
+              scheme: HTTP
+            initialDelaySeconds: 20
+            periodSeconds: 30
+            successThreshold: 1
+            failureThreshold: 10
+          {{- if .Values.volume.resources }}
+          resources:
+            {{ tpl .Values.volume.resources . | nindent 12 | trim }}
+          {{- end }}
+      volumes:
+        - name: seaweedfs-volume-log-volume
+          hostPath:
+            path: /storage/logs/seaweedfs/volume
+            type: DirectoryOrCreate
+        - name: seaweedfs-volume-storage
+          hostPath:
+            path: /storage/object_store/
+            type: DirectoryOrCreate
+        {{- if .Values.global.enableSecurity }}
+        - name: security-config
+          configMap:
+            name: {{ template "seaweedfs.name" . }}-security-config
+        - name: ca-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-ca-cert
+        - name: master-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-master-cert
+        - name: volume-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-volume-cert
+        - name: filer-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-filer-cert
+        - name: client-cert
+          secret:
+            secretName: {{ template "seaweedfs.name" . }}-client-cert
+        {{- end }}
+      {{- if .Values.volume.extraVolumes }}
+        {{ tpl .Values.volume.extraVolumes . | indent 8 | trim }}
+      {{- end }}
+      {{- if .Values.volume.nodeSelector }}
+      nodeSelector:
+        {{ tpl .Values.volume.nodeSelector . | indent 8 | trim }}
+      {{- end }}
+{{- end }}

k8s/seaweedfs/values.yaml

@@ -0,0 +1,308 @@
+# Available parameters and their default values for the SeaweedFS chart.
+
+global:
+  registry: ""
+  repository: ""
+  imageName: chrislusf/seaweedfs
+  imageTag: "1.61"
+  imagePullPolicy: IfNotPresent
+  imagePullSecrets: imagepullsecret
+  restartPolicy: Always
+  loggingLevel: 1
+  enableSecurity: false
+  monitoring:
+    enabled: false
+    gatewayHost: null
+    gatewayPort: null
+
+image:
+  registry: ""
+  repository: ""
+
+master:
+  enabled: true
+  repository: null
+  imageName: null
+  imageTag: null
+  imageOverride: null
+  restartPolicy: null
+  replicas: 1
+  port: 9333
+  grpcPort: 19333
+  ipBind: "0.0.0.0"
+  volumePreallocate: false
+  volumeSizeLimitMB: 30000
+  loggingOverrideLevel: null
+
+  # Disable http request, only gRpc operations are allowed
+  disableHttp: false
+
+  extraVolumes: ""
+  extraVolumeMounts: ""
+
+  # storage and storageClass are the settings for configuring stateful
+  # storage for the master pods. storage should be set to the disk size of
+  # the attached volume. storageClass is the class of storage which defaults
+  # to null (the Kube cluster will pick the default).
+  storage: 25Gi
+  storageClass: null
+
+  # Resource requests, limits, etc. for the master cluster placement. This
+  # should map directly to the value of the resources field for a PodSpec,
+  # formatted as a multi-line string. By default no direct resource request
+  # is made.
+  resources: null
+
+  # updatePartition is used to control a careful rolling update of SeaweedFS
+  # masters.
+  updatePartition: 0
+
+  # Affinity Settings
+  # Commenting out or setting as empty the affinity variable, will allow
+  # deployment to single node services such as Minikube
+  affinity: |
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        - labelSelector:
+            matchLabels:
+              app: {{ template "seaweedfs.name" . }}
+              release: "{{ .Release.Name }}"
+              component: master
+          topologyKey: kubernetes.io/hostname
+
+  # Toleration Settings for master pods
+  # This should be a multi-line string matching the Toleration array
+  # in a PodSpec.
+  tolerations: ""
+
+  # nodeSelector labels for master pod assignment, formatted as a muli-line string.
+  # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
+  # Example:
+  # nodeSelector: |
+  #   beta.kubernetes.io/arch: amd64
+  nodeSelector: |
+    sw-backend: "true"
+
+  # used to assign priority to master pods
+  # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+  priorityClassName: ""
+
+
+volume:
+  enabled: true
+  repository: null
+  imageName: null
+  imageTag: null
+  imageOverride: null
+  restartPolicy: null
+  port: 8080
+  grpcPort: 18080
+  ipBind: "0.0.0.0"
+  replicas: 1
+  loggingOverrideLevel: null
+
+  # limit background compaction or copying speed in mega bytes per second
+  compactionMBps: "40"
+
+  # Directories to store data files. dir[,dir]... (default "/tmp")
+  dir: "/data"
+
+  # Maximum numbers of volumes, count[,count]... (default "7")
+  maxVolumes: "10000"
+
+  # Volume server's rack name
+  rack: null
+
+  # Volume server's data center name
+  dataCenter: null
+
+  # Redirect moved or non-local volumes. (default true)
+  readRedirect: true
+
+  # Comma separated Ip addresses having write permission. No limit if empty.
+  whiteList: null
+
+  # Adjust jpg orientation when uploading.
+  imagesFixOrientation: false
+
+  extraVolumes: ""
+  extraVolumeMounts: ""
+
+  # Affinity Settings
+  # Commenting out or setting as empty the affinity variable, will allow
+  # deployment to single node services such as Minikube
+  affinity: |
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        - labelSelector:
+            matchLabels:
+              app: {{ template "seaweedfs.name" . }}
+              release: "{{ .Release.Name }}"
+              component: volume
+          topologyKey: kubernetes.io/hostname
+
+  # Resource requests, limits, etc. for the server cluster placement. This
+  # should map directly to the value of the resources field for a PodSpec,
+  # formatted as a multi-line string. By default no direct resource request
+  # is made.
+  resources: null
+
+  # Toleration Settings for server pods
+  # This should be a multi-line string matching the Toleration array
+  # in a PodSpec.
+  tolerations: ""
+
+  # nodeSelector labels for server pod assignment, formatted as a muli-line string.
+  # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
+  # Example:
+  # nodeSelector: |
+  #   beta.kubernetes.io/arch: amd64
+  nodeSelector: |
+    sw-volume: "true"
+
+  # used to assign priority to server pods
+  # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+  priorityClassName: ""
+
+
+filer:
+  enabled: true
+  repository: null
+  imageName: null
+  imageTag: null
+  imageOverride: null
+  restartPolicy: null
+  replicas: 1
+  port: 8888
+  grpcPort: 18888
+  loggingOverrideLevel: null
+
+  # Limit sub dir listing size (default 100000)
+  dirListLimit: 100000
+
+  # Turn off directory listing
+  disableDirListing: false
+
+  # Disable http request, only gRpc operations are allowed
+  disableHttp: false
+
+  # storage and storageClass are the settings for configuring stateful
+  # storage for the master pods. storage should be set to the disk size of
+  # the attached volume. storageClass is the class of storage which defaults
+  # to null (the Kube cluster will pick the default).
+  storage: 25Gi
+  storageClass: null
+
+  extraVolumes: ""
+  extraVolumeMounts: ""
+
+  # Affinity Settings
+  # Commenting out or setting as empty the affinity variable, will allow
+  # deployment to single node services such as Minikube
+  affinity: |
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        - labelSelector:
+            matchLabels:
+              app: {{ template "seaweedfs.name" . }}
+              release: "{{ .Release.Name }}"
+              component: filer
+          topologyKey: kubernetes.io/hostname
+
+  # updatePartition is used to control a careful rolling update of SeaweedFS
+  # masters.
+  updatePartition: 0
+
+  # Resource requests, limits, etc. for the server cluster placement. This
+  # should map directly to the value of the resources field for a PodSpec,
+  # formatted as a multi-line string. By default no direct resource request
+  # is made.
+  resources: null
+
+  # Toleration Settings for server pods
+  # This should be a multi-line string matching the Toleration array
+  # in a PodSpec.
+  tolerations: ""
+
+  # nodeSelector labels for server pod assignment, formatted as a muli-line string.
+  # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
+  # Example:
+  # nodeSelector: |
+  #   beta.kubernetes.io/arch: amd64
+  nodeSelector: |
+    sw-backend: "true"
+
+  # used to assign priority to server pods
+  # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+  priorityClassName: ""
+
+  dbSchema:
+    imageName: db-schema
+    imageTag: "development"
+    imageOverride: ""
+
+  # extraEnvVars is a list of extra enviroment variables to set with the stateful set.
+  extraEnvironmentVars:
+    WEED_MYSQL_ENABLED: "true"
+    WEED_MYSQL_HOSTNAME: "mysql-db-host"
+    WEED_MYSQL_PORT: "3306"
+    WEED_MYSQL_DATABASE: "sw_database"
+    WEED_MYSQL_CONNECTION_MAX_IDLE: "10"
+    WEED_MYSQL_CONNECTION_MAX_OPEN: "150"
+    # enable usage of memsql as filer backend
+    WEED_MYSQL_INTERPOLATEPARAMS: "true"
+    WEED_LEVELDB2_ENABLED: "false"
+    # with http DELETE, by default the filer would check whether a folder is empty.
+    # recursive_delete will delete all sub folders and files, similar to "rm -Rf"
+    WEED_FILER_OPTIONS_RECURSIVE_DELETE: "false"
+    # directories under this folder will be automatically creating a separate bucket
+    WEED_FILER_BUCKETS_FOLDER: "/buckets"
+    # directories under this folder will be store message queue data
+    WEED_FILER_QUEUES_FOLDER: "/queues"
+
+s3:
+  enabled: true
+  repository: null
+  imageName: null
+  imageTag: null
+  restartPolicy: null
+  replicas: 1
+  port: 8333
+  loggingOverrideLevel: null
+
+  # Suffix of the host name, {bucket}.{domainName}
+  domainName: ""
+
+  extraVolumes: ""
+  extraVolumeMounts: ""
+
+  # Resource requests, limits, etc. for the server cluster placement. This
+  # should map directly to the value of the resources field for a PodSpec,
+  # formatted as a multi-line string. By default no direct resource request
+  # is made.
+  resources: null
+
+  # Toleration Settings for server pods
+  # This should be a multi-line string matching the Toleration array
+  # in a PodSpec.
+  tolerations: ""
+
+  # nodeSelector labels for server pod assignment, formatted as a muli-line string.
+  # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
+  # Example:
+  # nodeSelector: |
+  #   beta.kubernetes.io/arch: amd64
+  nodeSelector: |
+    sw-backend: "true"
+
+  # used to assign priority to server pods
+  # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+  priorityClassName: ""
+
+certificates:
+  commonName: "SeaweedFS CA"
+  ipAddresses: []
+  keyAlgorithm: rsa
+  keySize: 2048
+  duration: 2160h # 90d
+  renewBefore: 360h # 15d

other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@@ -63,7 +63,7 @@ public class SeaweedRead {
         if (!chunkView.isFullChunk) {
             request.setHeader(HttpHeaders.ACCEPT_ENCODING, "");
             request.setHeader(HttpHeaders.RANGE,
-                String.format("bytes=%d-%d", chunkView.offset, chunkView.offset + chunkView.size));
+                String.format("bytes=%d-%d", chunkView.offset, chunkView.offset + chunkView.size - 1));
         }
 
         try {

other/java/client/src/main/proto/filer.proto

@@ -24,6 +24,9 @@ service SeaweedFiler {
     rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
     }
 
+    rpc StreamDeleteEntries (stream DeleteEntryRequest) returns (stream DeleteEntryResponse) {
+    }
+
     rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
     }
 
@@ -96,6 +99,8 @@ message FileChunk {
     string source_file_id = 6; // to be deprecated
     FileId fid = 7;
     FileId source_fid = 8;
+    bytes cipher_key = 9;
+    bool is_gzipped = 10;
 }
 
 message FileId {
@@ -123,9 +128,11 @@ message FuseAttributes {
 message CreateEntryRequest {
     string directory = 1;
     Entry entry = 2;
+    bool o_excl = 3;
 }
 
 message CreateEntryResponse {
+    string error = 1;
 }
 
 message UpdateEntryRequest {
@@ -145,6 +152,7 @@ message DeleteEntryRequest {
 }
 
 message DeleteEntryResponse {
+    string error = 1;
 }
 
 message AtomicRenameEntryRequest {
@@ -163,6 +171,7 @@ message AssignVolumeRequest {
     string replication = 3;
     int32 ttl_sec = 4;
     string data_center = 5;
+    string parent_path = 6;
 }
 
 message AssignVolumeResponse {
@@ -171,6 +180,9 @@ message AssignVolumeResponse {
     string public_url = 3;
     int32 count = 4;
     string auth = 5;
+    string collection = 6;
+    string replication = 7;
+    string error = 8;
 }
 
 message LookupVolumeRequest {
@@ -217,4 +229,7 @@ message GetFilerConfigurationResponse {
     string replication = 2;
     string collection = 3;
     uint32 max_mb = 4;
+    string dir_buckets = 5;
+    string dir_queues = 6;
+    bool cipher = 7;
 }

unmaintained/change_superblock/change_superblock.go

@@ -8,9 +8,9 @@ import (
 	"strconv"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
 )
 
 var (
@@ -51,7 +51,7 @@ func main() {
 	datBackend := backend.NewDiskFile(datFile)
 	defer datBackend.Close()
 
-	superBlock, err := storage.ReadSuperBlock(datBackend)
+	superBlock, err := super_block.ReadSuperBlock(datBackend)
 
 	if err != nil {
 		glog.Fatalf("cannot parse existing super block: %v", err)
@@ -63,7 +63,7 @@ func main() {
 	hasChange := false
 
 	if *targetReplica != "" {
-		replica, err := storage.NewReplicaPlacementFromString(*targetReplica)
+		replica, err := super_block.NewReplicaPlacementFromString(*targetReplica)
 
 		if err != nil {
 			glog.Fatalf("cannot parse target replica %s: %v", *targetReplica, err)

unmaintained/fix_dat/fix_dat.go

@@ -9,9 +9,9 @@ import (
 	"strconv"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
@@ -59,7 +59,7 @@ func main() {
 	}
 	defer newDatFile.Close()
 
-	superBlock, err := storage.ReadSuperBlock(datBackend)
+	superBlock, err := super_block.ReadSuperBlock(datBackend)
 	if err != nil {
 		glog.Fatalf("Read Volume Data superblock %v", err)
 	}
@@ -67,7 +67,7 @@ func main() {
 
 	iterateEntries(datBackend, indexFile, func(n *needle.Needle, offset int64) {
 		fmt.Printf("needle id=%v name=%s size=%d dataSize=%d\n", n.Id, string(n.Name), n.Size, n.DataSize)
-		_, s, _, e := n.Append(datBackend, superBlock.Version())
+		_, s, _, e := n.Append(datBackend, superBlock.Version)
 		fmt.Printf("size %d error %v\n", s, e)
 	})
 
@@ -81,13 +81,13 @@ func iterateEntries(datBackend backend.BackendStorageFile, idxFile *os.File, vis
 	readerOffset += int64(count)
 
 	// start to read dat file
-	superBlock, err := storage.ReadSuperBlock(datBackend)
+	superBlock, err := super_block.ReadSuperBlock(datBackend)
 	if err != nil {
 		fmt.Printf("cannot read dat file super block: %v", err)
 		return
 	}
 	offset := int64(superBlock.BlockSize())
-	version := superBlock.Version()
+	version := superBlock.Version
 	n, _, rest, err := needle.ReadNeedleHeader(datBackend, version, offset)
 	if err != nil {
 		fmt.Printf("cannot read needle header: %v", err)

unmaintained/remove_duplicate_fids/remove_duplicate_fids.go

@@ -10,6 +10,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
 )
 
 var (
@@ -24,7 +25,7 @@ func Checksum(n *needle.Needle) string {
 
 type VolumeFileScanner4SeeDat struct {
 	version needle.Version
-	block   storage.SuperBlock
+	block   super_block.SuperBlock
 
 	dir        string
 	hashes     map[string]bool
@@ -32,8 +33,8 @@ type VolumeFileScanner4SeeDat struct {
 	datBackend backend.BackendStorageFile
 }
 
-func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock storage.SuperBlock) error {
-	scanner.version = superBlock.Version()
+func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock super_block.SuperBlock) error {
+	scanner.version = superBlock.Version
 	scanner.block = superBlock
 	return nil
 

unmaintained/repeated_vacuum/repeated_vacuum.go

@@ -1,51 +1,60 @@
 package main
 
 import (
-	"bytes"
 	"flag"
 	"fmt"
 	"log"
 	"math/rand"
 
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/spf13/viper"
+	"google.golang.org/grpc"
 
 	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 var (
-	master = flag.String("master", "127.0.0.1:9333", "the master server")
-	repeat = flag.Int("n", 5, "repeat how many times")
+	master           = flag.String("master", "127.0.0.1:9333", "the master server")
+	repeat           = flag.Int("n", 5, "repeat how many times")
+	garbageThreshold = flag.Float64("garbageThreshold", 0.3, "garbageThreshold")
 )
 
 func main() {
 	flag.Parse()
 
 	util.LoadConfiguration("security", false)
-	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	genFile(grpcDialOption, 0)
 
 	for i := 0; i < *repeat; i++ {
-		assignResult, err := operation.Assign(*master, grpcDialOption, &operation.VolumeAssignRequest{Count: 1})
-		if err != nil {
-			log.Fatalf("assign: %v", err)
-		}
+		// create 2 files, and delete one of them
 
-		data := make([]byte, 1024)
-		rand.Read(data)
-		reader := bytes.NewReader(data)
-
-		targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
-
-		_, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), reader, false, "", nil, assignResult.Auth)
-		if err != nil {
-			log.Fatalf("upload: %v", err)
-		}
+		assignResult, targetUrl := genFile(grpcDialOption, i)
 
 		util.Delete(targetUrl, string(assignResult.Auth))
 
-		util.Get(fmt.Sprintf("http://%s/vol/vacuum", *master))
+		println("vacuum", i, "threshold", *garbageThreshold)
+		util.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=%f", *master, *garbageThreshold))
 
 	}
 
 }
+
+func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, string) {
+	assignResult, err := operation.Assign(*master, grpcDialOption, &operation.VolumeAssignRequest{Count: 1})
+	if err != nil {
+		log.Fatalf("assign: %v", err)
+	}
+
+	data := make([]byte, 1024)
+	rand.Read(data)
+
+	targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
+
+	_, err = operation.UploadData(targetUrl, fmt.Sprintf("test%d", i), false, data, false, "bench/test", nil, assignResult.Auth)
+	if err != nil {
+		log.Fatalf("upload: %v", err)
+	}
+	return assignResult, targetUrl
+}

unmaintained/see_dat/see_dat.go

@@ -7,6 +7,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
 )
 
 var (
@@ -19,8 +20,8 @@ type VolumeFileScanner4SeeDat struct {
 	version needle.Version
 }
 
-func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock storage.SuperBlock) error {
-	scanner.version = superBlock.Version()
+func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock super_block.SuperBlock) error {
+	scanner.version = superBlock.Version
 	return nil
 
 }

unmaintained/volume_tailer/volume_tailer.go

@@ -25,7 +25,7 @@ func main() {
 	flag.Parse()
 
 	util2.LoadConfiguration("security", false)
-	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	vid := needle.VolumeId(*volumeId)
 

weed/command/backup.go

@@ -5,8 +5,8 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/spf13/viper"
 
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/storage"
@@ -64,7 +64,7 @@ var cmdBackup = &Command{
 func runBackup(cmd *Command, args []string) bool {
 
 	util.LoadConfiguration("security", false)
-	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	if *s.volumeId == -1 {
 		return false
@@ -98,15 +98,15 @@ func runBackup(cmd *Command, args []string) bool {
 			return true
 		}
 	}
-	var replication *storage.ReplicaPlacement
+	var replication *super_block.ReplicaPlacement
 	if *s.replication != "" {
-		replication, err = storage.NewReplicaPlacementFromString(*s.replication)
+		replication, err = super_block.NewReplicaPlacementFromString(*s.replication)
 		if err != nil {
 			fmt.Printf("Error generate volume %d replication %s : %v\n", vid, *s.replication, err)
 			return true
 		}
 	} else {
-		replication, err = storage.NewReplicaPlacementFromString(stats.Replication)
+		replication, err = super_block.NewReplicaPlacementFromString(stats.Replication)
 		if err != nil {
 			fmt.Printf("Error get volume %d replication %s : %v\n", vid, stats.Replication, err)
 			return true
@@ -119,7 +119,7 @@ func runBackup(cmd *Command, args []string) bool {
 	}
 
 	if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
-		if err = v.Compact(0, 0); err != nil {
+		if err = v.Compact2(30 * 1024 * 1024 * 1024); err != nil {
 			fmt.Printf("Compact Volume before synchronizing %v\n", err)
 			return true
 		}

weed/command/benchmark.go

@@ -15,11 +15,11 @@ import (
 	"sync"
 	"time"
 
-	"github.com/spf13/viper"
 	"google.golang.org/grpc"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
@@ -41,6 +41,7 @@ type BenchmarkOptions struct {
 	maxCpu           *int
 	grpcDialOption   grpc.DialOption
 	masterClient     *wdclient.MasterClient
+	grpcRead         *bool
 }
 
 var (
@@ -65,6 +66,7 @@ func init() {
 	b.replication = cmdBenchmark.Flag.String("replication", "000", "replication type")
 	b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
 	b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
+	b.grpcRead = cmdBenchmark.Flag.Bool("grpcRead", false, "use grpc API to read")
 	sharedBytes = make([]byte, 1024)
 }
 
@@ -109,7 +111,7 @@ var (
 func runBenchmark(cmd *Command, args []string) bool {
 
 	util.LoadConfiguration("security", false)
-	b.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
+	b.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
 	if *b.maxCpu < 1 {
@@ -125,7 +127,7 @@ func runBenchmark(cmd *Command, args []string) bool {
 		defer pprof.StopCPUProfile()
 	}
 
-	b.masterClient = wdclient.NewMasterClient(context.Background(), b.grpcDialOption, "client", strings.Split(*b.masters, ","))
+	b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", 0, strings.Split(*b.masters, ","))
 	go b.masterClient.KeepConnectedToMaster()
 	b.masterClient.WaitUntilConnected()
 
@@ -279,23 +281,61 @@ func readFiles(fileIdLineChan chan string, s *stat) {
 			fmt.Printf("reading file %s\n", fid)
 		}
 		start := time.Now()
-		url, err := b.masterClient.LookupFileId(fid)
-		if err != nil {
-			s.failed++
-			println("!!!! ", fid, " location not found!!!!!")
-			continue
+		var bytesRead int
+		var err error
+		if *b.grpcRead {
+			volumeServer, err := b.masterClient.LookupVolumeServer(fid)
+			if err != nil {
+				s.failed++
+				println("!!!! ", fid, " location not found!!!!!")
+				continue
+			}
+			bytesRead, err = grpcFileGet(volumeServer, fid, b.grpcDialOption)
+		} else {
+			url, err := b.masterClient.LookupFileId(fid)
+			if err != nil {
+				s.failed++
+				println("!!!! ", fid, " location not found!!!!!")
+				continue
+			}
+			var bytes []byte
+			bytes, err = util.Get(url)
+			bytesRead = len(bytes)
 		}
-		if bytesRead, err := util.Get(url); err == nil {
+		if err == nil {
 			s.completed++
-			s.transferred += int64(len(bytesRead))
+			s.transferred += int64(bytesRead)
 			readStats.addSample(time.Now().Sub(start))
 		} else {
 			s.failed++
-			fmt.Printf("Failed to read %s error:%v\n", url, err)
+			fmt.Printf("Failed to read %s error:%v\n", fid, err)
 		}
 	}
 }
 
+func grpcFileGet(volumeServer, fid string, grpcDialOption grpc.DialOption) (bytesRead int, err error) {
+	err = operation.WithVolumeServerClient(volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+		fileGetClient, err := client.FileGet(context.Background(), &volume_server_pb.FileGetRequest{FileId: fid})
+		if err != nil {
+			return err
+		}
+
+		for {
+			resp, respErr := fileGetClient.Recv()
+			if resp != nil {
+				bytesRead += len(resp.Data)
+			}
+			if respErr != nil {
+				if respErr == io.EOF {
+					return nil
+				}
+				return respErr
+			}
+		}
+	})
+	return
+}
+
 func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan bool) {
 	file, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
 	if err != nil {

weed/command/command.go

@@ -20,6 +20,7 @@ var Commands = []*Command{
 	cmdS3,
 	cmdUpload,
 	cmdDownload,
+	cmdMsgBroker,
 	cmdScaffold,
 	cmdShell,
 	cmdVersion,

weed/command/compact.go

@@ -17,6 +17,9 @@ var cmdCompact = &Command{
   The compacted .dat file is stored as .cpd file.
   The compacted .idx file is stored as .cpx file.
 
+  For method=0, it compacts based on the .dat file, works if .idx file is corrupted.
+  For method=1, it compacts based on the .idx file, works if deletion happened but not written to .dat files.
+
   `,
 }
 
@@ -47,7 +50,7 @@ func runCompact(cmd *Command, args []string) bool {
 			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	} else {
-		if err = v.Compact2(); err != nil {
+		if err = v.Compact2(preallocate); err != nil {
 			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	}

weed/command/download.go

@@ -71,6 +71,7 @@ func downloadToFile(server, fileId, saveDir string) error {
 	}
 	f, err := os.OpenFile(path.Join(saveDir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
 	if err != nil {
+		io.Copy(ioutil.Discard, rc)
 		return err
 	}
 	defer f.Close()

weed/command/export.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"fmt"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
@@ -12,11 +13,11 @@ import (
 	"text/template"
 	"time"
 
-	"io"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
+	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 
@@ -89,12 +90,12 @@ func printNeedle(vid needle.VolumeId, n *needle.Needle, version needle.Version,
 type VolumeFileScanner4Export struct {
 	version   needle.Version
 	counter   int
-	needleMap *storage.NeedleMap
+	needleMap *needle_map.MemDb
 	vid       needle.VolumeId
 }
 
-func (scanner *VolumeFileScanner4Export) VisitSuperBlock(superBlock storage.SuperBlock) error {
-	scanner.version = superBlock.Version()
+func (scanner *VolumeFileScanner4Export) VisitSuperBlock(superBlock super_block.SuperBlock) error {
+	scanner.version = superBlock.Version
 	return nil
 
 }
@@ -192,15 +193,12 @@ func runExport(cmd *Command, args []string) bool {
 		fileName = *export.collection + "_" + fileName
 	}
 	vid := needle.VolumeId(*export.volumeId)
-	indexFile, err := os.OpenFile(path.Join(*export.dir, fileName+".idx"), os.O_RDONLY, 0644)
-	if err != nil {
-		glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
-	}
-	defer indexFile.Close()
 
-	needleMap, err := storage.LoadBtreeNeedleMap(indexFile)
-	if err != nil {
-		glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
+	needleMap := needle_map.NewMemDb()
+	defer needleMap.Close()
+
+	if err := needleMap.LoadFromIdx(path.Join(*export.dir, fileName+".idx")); err != nil {
+		glog.Fatalf("cannot load needle map from %s.idx: %s", fileName, err)
 	}
 
 	volumeFileScanner := &VolumeFileScanner4Export{

weed/command/filer.go

@@ -6,14 +6,14 @@ import (
 	"strings"
 	"time"
 
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/spf13/viper"
+	"google.golang.org/grpc/reflection"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"google.golang.org/grpc/reflection"
 )
 
 var (
@@ -27,13 +27,13 @@ type FilerOptions struct {
 	publicPort              *int
 	collection              *string
 	defaultReplicaPlacement *string
-	redirectOnRead          *bool
 	disableDirListing       *bool
 	maxMB                   *int
 	dirListingLimit         *int
 	dataCenter              *string
 	enableNotification      *bool
 	disableHttp             *bool
+	cipher                  *bool
 
 	// default leveldb directory, used in "weed server" mode
 	defaultLevelDbDirectory *string
@@ -47,12 +47,12 @@ func init() {
 	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
 	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
 	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
-	f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
 	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
 	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
 	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center")
 	f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
+	f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
 }
 
 var cmdFiler = &Command{
@@ -103,14 +103,14 @@ func (fo *FilerOptions) startFiler() {
 		Masters:            strings.Split(*fo.masters, ","),
 		Collection:         *fo.collection,
 		DefaultReplication: *fo.defaultReplicaPlacement,
-		RedirectOnRead:     *fo.redirectOnRead,
 		DisableDirListing:  *fo.disableDirListing,
 		MaxMB:              *fo.maxMB,
 		DirListingLimit:    *fo.dirListingLimit,
 		DataCenter:         *fo.dataCenter,
 		DefaultLevelDbDir:  defaultLevelDbDirectory,
 		DisableHttp:        *fo.disableHttp,
-		Port:               *fo.port,
+		Port:               uint32(*fo.port),
+		Cipher:             *fo.cipher,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)
@@ -145,7 +145,7 @@ func (fo *FilerOptions) startFiler() {
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
 	}
-	grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "filer"))
+	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.filer"))
 	filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
 	reflection.Register(grpcS)
 	go grpcS.Serve(grpcL)

weed/command/filer_copy.go

@@ -14,13 +14,15 @@ import (
 	"sync"
 	"time"
 
+	"google.golang.org/grpc"
+
 	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
-	"github.com/spf13/viper"
-	"google.golang.org/grpc"
 )
 
 var (
@@ -37,9 +39,10 @@ type CopyOptions struct {
 	masterClient      *wdclient.MasterClient
 	concurrenctFiles  *int
 	concurrenctChunks *int
-	compressionLevel  *int
 	grpcDialOption    grpc.DialOption
 	masters           []string
+	cipher            bool
+	ttlSec            int32
 }
 
 func init() {
@@ -52,7 +55,6 @@ func init() {
 	copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit")
 	copy.concurrenctFiles = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
 	copy.concurrenctChunks = cmdCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
-	copy.compressionLevel = cmdCopy.Flag.Int("compressionLevel", 9, "local file compression level 1 ~ 9")
 }
 
 var cmdCopy = &Command{
@@ -105,11 +107,9 @@ func runCopy(cmd *Command, args []string) bool {
 
 	filerGrpcPort := filerPort + 10000
 	filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort)
-	copy.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
+	copy.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
 
-	ctx := context.Background()
-
-	masters, collection, replication, maxMB, err := readFilerConfiguration(ctx, copy.grpcDialOption, filerGrpcAddress)
+	masters, collection, replication, maxMB, cipher, err := readFilerConfiguration(copy.grpcDialOption, filerGrpcAddress)
 	if err != nil {
 		fmt.Printf("read from filer %s: %v\n", filerGrpcAddress, err)
 		return false
@@ -124,10 +124,14 @@ func runCopy(cmd *Command, args []string) bool {
 		*copy.maxMB = int(maxMB)
 	}
 	copy.masters = masters
+	copy.cipher = cipher
 
-	copy.masterClient = wdclient.NewMasterClient(ctx, copy.grpcDialOption, "client", copy.masters)
-	go copy.masterClient.KeepConnectedToMaster()
-	copy.masterClient.WaitUntilConnected()
+	ttl, err := needle.ReadTTL(*copy.ttl)
+	if err != nil {
+		fmt.Printf("parsing ttl %s: %v\n", *copy.ttl, err)
+		return false
+	}
+	copy.ttlSec = int32(ttl.Minutes()) * 60
 
 	if *cmdCopy.IsDebug {
 		util.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof")
@@ -153,7 +157,7 @@ func runCopy(cmd *Command, args []string) bool {
 				filerHost:        filerUrl.Host,
 				filerGrpcAddress: filerGrpcAddress,
 			}
-			if err := worker.copyFiles(ctx, fileCopyTaskChan); err != nil {
+			if err := worker.copyFiles(fileCopyTaskChan); err != nil {
 				fmt.Fprintf(os.Stderr, "copy file error: %v\n", err)
 				return
 			}
@@ -164,13 +168,14 @@ func runCopy(cmd *Command, args []string) bool {
 	return true
 }
 
-func readFilerConfiguration(ctx context.Context, grpcDialOption grpc.DialOption, filerGrpcAddress string) (masters []string, collection, replication string, maxMB uint32, err error) {
-	err = withFilerClient(ctx, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-		resp, err := client.GetFilerConfiguration(ctx, &filer_pb.GetFilerConfigurationRequest{})
+func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress string) (masters []string, collection, replication string, maxMB uint32, cipher bool, err error) {
+	err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 		if err != nil {
 			return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
 		}
 		masters, collection, replication, maxMB = resp.Masters, resp.Collection, resp.Replication, resp.MaxMb
+		cipher = resp.Cipher
 		return nil
 	})
 	return
@@ -215,9 +220,9 @@ type FileCopyWorker struct {
 	filerGrpcAddress string
 }
 
-func (worker *FileCopyWorker) copyFiles(ctx context.Context, fileCopyTaskChan chan FileCopyTask) error {
+func (worker *FileCopyWorker) copyFiles(fileCopyTaskChan chan FileCopyTask) error {
 	for task := range fileCopyTaskChan {
-		if err := worker.doEachCopy(ctx, task); err != nil {
+		if err := worker.doEachCopy(task); err != nil {
 			return err
 		}
 	}
@@ -233,7 +238,7 @@ type FileCopyTask struct {
 	gid                uint32
 }
 
-func (worker *FileCopyWorker) doEachCopy(ctx context.Context, task FileCopyTask) error {
+func (worker *FileCopyWorker) doEachCopy(task FileCopyTask) error {
 
 	f, err := os.Open(task.sourceLocation)
 	if err != nil {
@@ -261,36 +266,55 @@ func (worker *FileCopyWorker) doEachCopy(ctx context.Context, task FileCopyTask)
 	}
 
 	if chunkCount == 1 {
-		return worker.uploadFileAsOne(ctx, task, f)
+		return worker.uploadFileAsOne(task, f)
 	}
 
-	return worker.uploadFileInChunks(ctx, task, f, chunkCount, chunkSize)
+	return worker.uploadFileInChunks(task, f, chunkCount, chunkSize)
 }
 
-func (worker *FileCopyWorker) uploadFileAsOne(ctx context.Context, task FileCopyTask, f *os.File) error {
+func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) error {
 
 	// upload the file content
 	fileName := filepath.Base(f.Name())
 	mimeType := detectMimeType(f)
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return err
+	}
 
 	var chunks []*filer_pb.FileChunk
+	var assignResult *filer_pb.AssignVolumeResponse
+	var assignError error
 
 	if task.fileSize > 0 {
 
 		// assign a volume
-		assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
-			Count:       1,
-			Replication: *worker.options.replication,
-			Collection:  *worker.options.collection,
-			Ttl:         *worker.options.ttl,
+		err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+
+			request := &filer_pb.AssignVolumeRequest{
+				Count:       1,
+				Replication: *worker.options.replication,
+				Collection:  *worker.options.collection,
+				TtlSec:      worker.options.ttlSec,
+				ParentPath:  task.destinationUrlPath,
+			}
+
+			assignResult, assignError = client.AssignVolume(context.Background(), request)
+			if assignError != nil {
+				return fmt.Errorf("assign volume failure %v: %v", request, assignError)
+			}
+			if assignResult.Error != "" {
+				return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error)
+			}
+			return nil
 		})
 		if err != nil {
 			fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err)
 		}
 
-		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
+		targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId
 
-		uploadResult, err := operation.UploadWithLocalCompressionLevel(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth, *worker.options.compressionLevel)
+		uploadResult, err := operation.UploadData(targetUrl, fileName, worker.options.cipher, data, false, mimeType, nil, security.EncodedJwt(assignResult.Auth))
 		if err != nil {
 			return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
 		}
@@ -300,17 +324,19 @@ func (worker *FileCopyWorker) uploadFileAsOne(ctx context.Context, task FileCopy
 		fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
 
 		chunks = append(chunks, &filer_pb.FileChunk{
-			FileId: assignResult.Fid,
-			Offset: 0,
-			Size:   uint64(uploadResult.Size),
-			Mtime:  time.Now().UnixNano(),
-			ETag:   uploadResult.ETag,
+			FileId:    assignResult.FileId,
+			Offset:    0,
+			Size:      uint64(uploadResult.Size),
+			Mtime:     time.Now().UnixNano(),
+			ETag:      uploadResult.Md5,
+			CipherKey: uploadResult.CipherKey,
+			IsGzipped: uploadResult.Gzip > 0,
 		})
 
 		fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
 	}
 
-	if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
@@ -325,13 +351,13 @@ func (worker *FileCopyWorker) uploadFileAsOne(ctx context.Context, task FileCopy
 					Mime:        mimeType,
 					Replication: *worker.options.replication,
 					Collection:  *worker.options.collection,
-					TtlSec:      int32(util.ParseInt(*worker.options.ttl, 0)),
+					TtlSec:      worker.options.ttlSec,
 				},
 				Chunks: chunks,
 			},
 		}
 
-		if _, err := client.CreateEntry(ctx, request); err != nil {
+		if err := filer_pb.CreateEntry(client, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)
 		}
 		return nil
@@ -342,7 +368,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(ctx context.Context, task FileCopy
 	return nil
 }
 
-func (worker *FileCopyWorker) uploadFileInChunks(ctx context.Context, task FileCopyTask, f *os.File, chunkCount int, chunkSize int64) error {
+func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, chunkCount int, chunkSize int64) error {
 
 	fileName := filepath.Base(f.Name())
 	mimeType := detectMimeType(f)
@@ -352,6 +378,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(ctx context.Context, task FileC
 	concurrentChunks := make(chan struct{}, *worker.options.concurrenctChunks)
 	var wg sync.WaitGroup
 	var uploadError error
+	var collection, replication string
 
 	fmt.Printf("uploading %s in %d chunks ...\n", fileName, chunkCount)
 	for i := int64(0); i < int64(chunkCount) && uploadError == nil; i++ {
@@ -363,22 +390,42 @@ func (worker *FileCopyWorker) uploadFileInChunks(ctx context.Context, task FileC
 				<-concurrentChunks
 			}()
 			// assign a volume
-			assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
-				Count:       1,
-				Replication: *worker.options.replication,
-				Collection:  *worker.options.collection,
-				Ttl:         *worker.options.ttl,
+			var assignResult *filer_pb.AssignVolumeResponse
+			var assignError error
+			err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+				request := &filer_pb.AssignVolumeRequest{
+					Count:       1,
+					Replication: *worker.options.replication,
+					Collection:  *worker.options.collection,
+					TtlSec:      worker.options.ttlSec,
+					ParentPath:  task.destinationUrlPath,
+				}
+
+				assignResult, assignError = client.AssignVolume(context.Background(), request)
+				if assignError != nil {
+					return fmt.Errorf("assign volume failure %v: %v", request, assignError)
+				}
+				if assignResult.Error != "" {
+					return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error)
+				}
+				return nil
 			})
 			if err != nil {
 				fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err)
 			}
+			if err != nil {
+				fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err)
+			}
 
-			targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
+			targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId
+			if collection == "" {
+				collection = assignResult.Collection
+			}
+			if replication == "" {
+				replication = assignResult.Replication
+			}
 
-			uploadResult, err := operation.Upload(targetUrl,
-				fileName+"-"+strconv.FormatInt(i+1, 10),
-				io.NewSectionReader(f, i*chunkSize, chunkSize),
-				false, "", nil, assignResult.Auth)
+			uploadResult, err := operation.Upload(targetUrl, fileName+"-"+strconv.FormatInt(i+1, 10), worker.options.cipher, io.NewSectionReader(f, i*chunkSize, chunkSize), false, "", nil, security.EncodedJwt(assignResult.Auth))
 			if err != nil {
 				uploadError = fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
 				return
@@ -388,11 +435,13 @@ func (worker *FileCopyWorker) uploadFileInChunks(ctx context.Context, task FileC
 				return
 			}
 			chunksChan <- &filer_pb.FileChunk{
-				FileId: assignResult.Fid,
-				Offset: i * chunkSize,
-				Size:   uint64(uploadResult.Size),
-				Mtime:  time.Now().UnixNano(),
-				ETag:   uploadResult.ETag,
+				FileId:    assignResult.FileId,
+				Offset:    i * chunkSize,
+				Size:      uint64(uploadResult.Size),
+				Mtime:     time.Now().UnixNano(),
+				ETag:      uploadResult.ETag,
+				CipherKey: uploadResult.CipherKey,
+				IsGzipped: uploadResult.Gzip > 0,
 			}
 			fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
 		}(i)
@@ -410,11 +459,11 @@ func (worker *FileCopyWorker) uploadFileInChunks(ctx context.Context, task FileC
 		for _, chunk := range chunks {
 			fileIds = append(fileIds, chunk.FileId)
 		}
-		operation.DeleteFiles(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, fileIds)
+		operation.DeleteFiles(copy.masters[0], worker.options.grpcDialOption, fileIds)
 		return uploadError
 	}
 
-	if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
@@ -427,15 +476,15 @@ func (worker *FileCopyWorker) uploadFileInChunks(ctx context.Context, task FileC
 					FileSize:    uint64(task.fileSize),
 					FileMode:    uint32(task.fileMode),
 					Mime:        mimeType,
-					Replication: *worker.options.replication,
-					Collection:  *worker.options.collection,
-					TtlSec:      int32(util.ParseInt(*worker.options.ttl, 0)),
+					Replication: replication,
+					Collection:  collection,
+					TtlSec:      worker.options.ttlSec,
 				},
 				Chunks: chunks,
 			},
 		}
 
-		if _, err := client.CreateEntry(ctx, request); err != nil {
+		if err := filer_pb.CreateEntry(client, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)
 		}
 		return nil
@@ -457,18 +506,12 @@ func detectMimeType(f *os.File) string {
 	}
 	if err != nil {
 		fmt.Printf("read head of %v: %v\n", f.Name(), err)
-		return "application/octet-stream"
+		return ""
 	}
 	f.Seek(0, io.SeekStart)
 	mimeType := http.DetectContentType(head[:n])
+	if mimeType == "application/octet-stream" {
+		return ""
+	}
 	return mimeType
 }
-
-func withFilerClient(ctx context.Context, filerAddress string, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
-
-	return util.WithCachedGrpcClient(ctx, func(clientConn *grpc.ClientConn) error {
-		client := filer_pb.NewSeaweedFilerClient(clientConn)
-		return fn(client)
-	}, filerAddress, grpcDialOption)
-
-}

weed/command/filer_replication.go

@@ -39,7 +39,7 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	util.LoadConfiguration("replication", true)
 	util.LoadConfiguration("notification", true)
-	config := viper.GetViper()
+	config := util.GetViper()
 
 	var notificationInput sub.NotificationInput
 
@@ -47,8 +47,7 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 
 	for _, input := range sub.NotificationInputs {
 		if config.GetBool("notification." + input.GetName() + ".enabled") {
-			viperSub := config.Sub("notification." + input.GetName())
-			if err := input.Initialize(viperSub); err != nil {
+			if err := input.Initialize(config, "notification."+input.GetName()+"."); err != nil {
 				glog.Fatalf("Failed to initialize notification input for %s: %+v",
 					input.GetName(), err)
 			}
@@ -66,10 +65,9 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 
 	// avoid recursive replication
 	if config.GetBool("notification.source.filer.enabled") && config.GetBool("notification.sink.filer.enabled") {
-		sourceConfig, sinkConfig := config.Sub("source.filer"), config.Sub("sink.filer")
-		if sourceConfig.GetString("grpcAddress") == sinkConfig.GetString("grpcAddress") {
-			fromDir := sourceConfig.GetString("directory")
-			toDir := sinkConfig.GetString("directory")
+		if config.GetString("source.filer.grpcAddress") == config.GetString("sink.filer.grpcAddress") {
+			fromDir := config.GetString("source.filer.directory")
+			toDir := config.GetString("sink.filer.directory")
 			if strings.HasPrefix(toDir, fromDir) {
 				glog.Fatalf("recursive replication! source directory %s includes the sink directory %s", fromDir, toDir)
 			}
@@ -79,8 +77,7 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 	var dataSink sink.ReplicationSink
 	for _, sk := range sink.Sinks {
 		if config.GetBool("sink." + sk.GetName() + ".enabled") {
-			viperSub := config.Sub("sink." + sk.GetName())
-			if err := sk.Initialize(viperSub); err != nil {
+			if err := sk.Initialize(config, "sink."+sk.GetName()+"."); err != nil {
 				glog.Fatalf("Failed to initialize sink for %s: %+v",
 					sk.GetName(), err)
 			}
@@ -98,7 +95,7 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 		return true
 	}
 
-	replicator := replication.NewReplicator(config.Sub("source.filer"), dataSink)
+	replicator := replication.NewReplicator(config, "source.filer.", dataSink)
 
 	for {
 		key, m, err := notificationInput.ReceiveMessage()

weed/command/fix.go

@@ -8,6 +8,8 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
+	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 
@@ -31,11 +33,11 @@ var (
 
 type VolumeFileScanner4Fix struct {
 	version needle.Version
-	nm      *storage.NeedleMap
+	nm      *needle_map.MemDb
 }
 
-func (scanner *VolumeFileScanner4Fix) VisitSuperBlock(superBlock storage.SuperBlock) error {
-	scanner.version = superBlock.Version()
+func (scanner *VolumeFileScanner4Fix) VisitSuperBlock(superBlock super_block.SuperBlock) error {
+	scanner.version = superBlock.Version
 	return nil
 
 }
@@ -46,11 +48,11 @@ func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
 func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
 	glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped())
 	if n.Size > 0 && n.Size != types.TombstoneFileSize {
-		pe := scanner.nm.Put(n.Id, types.ToOffset(offset), n.Size)
+		pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size)
 		glog.V(2).Infof("saved %d with error %v", n.Size, pe)
 	} else {
 		glog.V(2).Infof("skipping deleted file ...")
-		return scanner.nm.Delete(n.Id, types.ToOffset(offset))
+		return scanner.nm.Delete(n.Id)
 	}
 	return nil
 }
@@ -66,13 +68,8 @@ func runFix(cmd *Command, args []string) bool {
 		baseFileName = *fixVolumeCollection + "_" + baseFileName
 	}
 	indexFileName := path.Join(*fixVolumePath, baseFileName+".idx")
-	indexFile, err := os.OpenFile(indexFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
-	if err != nil {
-		glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
-	}
-	defer indexFile.Close()
 
-	nm := storage.NewBtreeNeedleMap(indexFile)
+	nm := needle_map.NewMemDb()
 	defer nm.Close()
 
 	vid := needle.VolumeId(*fixVolumeId)
@@ -80,9 +77,13 @@ func runFix(cmd *Command, args []string) bool {
 		nm: nm,
 	}
 
-	err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid, storage.NeedleMapInMemory, scanner)
-	if err != nil {
-		glog.Fatalf("Export Volume File [ERROR] %s\n", err)
+	if err := storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid, storage.NeedleMapInMemory, scanner); err != nil {
+		glog.Fatalf("scan .dat File: %v", err)
+		os.Remove(indexFileName)
+	}
+
+	if err := nm.SaveToIdx(indexFileName); err != nil {
+		glog.Fatalf("save to .idx File: %v", err)
 		os.Remove(indexFileName)
 	}
 

weed/command/master.go

@@ -8,15 +8,16 @@ import (
 	"strings"
 
 	"github.com/chrislusf/raft/protobuf"
+	"github.com/gorilla/mux"
+	"google.golang.org/grpc/reflection"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/gorilla/mux"
-	"github.com/spf13/viper"
-	"google.golang.org/grpc/reflection"
 )
 
 var (
@@ -102,7 +103,7 @@ func runMaster(cmd *Command, args []string) bool {
 
 func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 
-	backend.LoadConfiguration(viper.GetViper())
+	backend.LoadConfiguration(util.GetViper())
 
 	myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.peers)
 
@@ -115,7 +116,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		glog.Fatalf("Master startup error: %v", e)
 	}
 	// start raftServer
-	raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
+	raftServer := weed_server.NewRaftServer(security.LoadClientTLS(util.GetViper(), "grpc.master"),
 		peers, myMasterAddress, *masterOption.metaFolder, ms.Topo, *masterOption.pulseSeconds)
 	if raftServer == nil {
 		glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717", *masterOption.metaFolder)
@@ -129,7 +130,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
 	}
 	// Create your protocol servers.
-	grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
+	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
 	master_pb.RegisterSeaweedServer(grpcS, ms)
 	protobuf.RegisterRaftServer(grpcS, raftServer)
 	reflection.Register(grpcS)

weed/command/mount.go

@@ -1,23 +1,18 @@
 package command
 
-import (
-	"fmt"
-	"strconv"
-	"strings"
-)
-
 type MountOptions struct {
-	filer              *string
-	filerMountRootPath *string
-	dir                *string
-	dirListCacheLimit  *int64
-	collection         *string
-	replication        *string
-	ttlSec             *int
-	chunkSizeLimitMB   *int
-	dataCenter         *string
-	allowOthers        *bool
-	umaskString        *string
+	filer                       *string
+	filerMountRootPath          *string
+	dir                         *string
+	dirListCacheLimit           *int64
+	collection                  *string
+	replication                 *string
+	ttlSec                      *int
+	chunkSizeLimitMB            *int
+	dataCenter                  *string
+	allowOthers                 *bool
+	umaskString                 *string
+	outsideContainerClusterMode *bool
 }
 
 var (
@@ -41,6 +36,7 @@ func init() {
 	mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")
 	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
 	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
+	mountOptions.outsideContainerClusterMode = cmdMount.Flag.Bool("outsideContainerClusterMode", false, "allows other users to access the file system")
 }
 
 var cmdMount = &Command{
@@ -58,21 +54,11 @@ var cmdMount = &Command{
 
   On OS X, it requires OSXFUSE (http://osxfuse.github.com/).
 
+  If the SeaweedFS system runs in a container cluster, e.g. managed by kubernetes or docker compose,
+  the volume servers are not accessible by their own ip addresses. 
+  In "outsideContainerClusterMode", the mount will use the filer ip address instead, assuming:
+    * All volume server containers are accessible through the same hostname or IP address as the filer.
+    * All volume server container ports are open external to the cluster.
+
   `,
 }
-
-func parseFilerGrpcAddress(filer string) (filerGrpcAddress string, err error) {
-	hostnameAndPort := strings.Split(filer, ":")
-	if len(hostnameAndPort) != 2 {
-		return "", fmt.Errorf("filer should have hostname:port format: %v", hostnameAndPort)
-	}
-
-	filerPort, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64)
-	if parseErr != nil {
-		return "", fmt.Errorf("filer port parse error: %v", parseErr)
-	}
-
-	filerGrpcPort := int(filerPort) + 10000
-
-	return fmt.Sprintf("%s:%d", hostnameAndPort[0], filerGrpcPort), nil
-}

weed/command/mount_std.go

@@ -3,6 +3,7 @@
 package command
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"os/user"
@@ -13,10 +14,11 @@ import (
 	"time"
 
 	"github.com/jacobsa/daemonize"
-	"github.com/spf13/viper"
 
 	"github.com/chrislusf/seaweedfs/weed/filesys"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/seaweedfs/fuse"
@@ -45,11 +47,12 @@ func runMount(cmd *Command, args []string) bool {
 		*mountOptions.ttlSec,
 		*mountOptions.dirListCacheLimit,
 		os.FileMode(umask),
+		*mountOptions.outsideContainerClusterMode,
 	)
 }
 
 func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCenter string, chunkSizeLimitMB int,
-	allowOthers bool, ttlSec int, dirListCacheLimit int64, umask os.FileMode) bool {
+	allowOthers bool, ttlSec int, dirListCacheLimit int64, umask os.FileMode, outsideContainerClusterMode bool) bool {
 
 	util.LoadConfiguration("security", false)
 
@@ -100,7 +103,7 @@ func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCente
 		fuse.VolumeName(mountName),
 		fuse.FSName(filer + ":" + filerMountRootPath),
 		fuse.Subtype("seaweedfs"),
-		fuse.NoAppleDouble(),
+		// fuse.NoAppleDouble(), // include .DS_Store, otherwise can not delete non-empty folders
 		fuse.NoAppleXattr(),
 		fuse.NoBrowse(),
 		fuse.AutoXattr(),
@@ -132,13 +135,31 @@ func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCente
 		c.Close()
 	})
 
-	filerGrpcAddress, err := parseFilerGrpcAddress(filer)
+	// parse filer grpc address
+	filerGrpcAddress, err := pb.ParseFilerGrpcAddress(filer)
 	if err != nil {
-		glog.V(0).Infof("parseFilerGrpcAddress: %v", err)
+		glog.V(0).Infof("ParseFilerGrpcAddress: %v", err)
 		daemonize.SignalOutcome(err)
 		return true
 	}
 
+	// try to connect to filer, filerBucketsPath may be useful later
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+	var cipher bool
+	err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+		if err != nil {
+			return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
+		}
+		cipher = resp.Cipher
+		return nil
+	})
+	if err != nil {
+		glog.Fatal(err)
+		return false
+	}
+
+	// find mount point
 	mountRoot := filerMountRootPath
 	if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") {
 		mountRoot = mountRoot[0 : len(mountRoot)-1]
@@ -147,22 +168,24 @@ func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCente
 	daemonize.SignalOutcome(nil)
 
 	err = fs.Serve(c, filesys.NewSeaweedFileSystem(&filesys.Option{
-		FilerGrpcAddress:   filerGrpcAddress,
-		GrpcDialOption:     security.LoadClientTLS(viper.Sub("grpc"), "client"),
-		FilerMountRootPath: mountRoot,
-		Collection:         collection,
-		Replication:        replication,
-		TtlSec:             int32(ttlSec),
-		ChunkSizeLimit:     int64(chunkSizeLimitMB) * 1024 * 1024,
-		DataCenter:         dataCenter,
-		DirListCacheLimit:  dirListCacheLimit,
-		EntryCacheTtl:      3 * time.Second,
-		MountUid:           uid,
-		MountGid:           gid,
-		MountMode:          mountMode,
-		MountCtime:         fileInfo.ModTime(),
-		MountMtime:         time.Now(),
-		Umask:              umask,
+		FilerGrpcAddress:            filerGrpcAddress,
+		GrpcDialOption:              grpcDialOption,
+		FilerMountRootPath:          mountRoot,
+		Collection:                  collection,
+		Replication:                 replication,
+		TtlSec:                      int32(ttlSec),
+		ChunkSizeLimit:              int64(chunkSizeLimitMB) * 1024 * 1024,
+		DataCenter:                  dataCenter,
+		DirListCacheLimit:           dirListCacheLimit,
+		EntryCacheTtl:               3 * time.Second,
+		MountUid:                    uid,
+		MountGid:                    gid,
+		MountMode:                   mountMode,
+		MountCtime:                  fileInfo.ModTime(),
+		MountMtime:                  time.Now(),
+		Umask:                       umask,
+		OutsideContainerClusterMode: outsideContainerClusterMode,
+		Cipher:                      cipher,
 	}))
 	if err != nil {
 		fuse.Unmount(dir)

weed/command/msg_broker.go

@@ -0,0 +1,107 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"google.golang.org/grpc/reflection"
+
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/queue_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	weed_server "github.com/chrislusf/seaweedfs/weed/server"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+var (
+	messageBrokerStandaloneOptions QueueOptions
+)
+
+type QueueOptions struct {
+	filer      *string
+	port       *int
+	defaultTtl *string
+}
+
+func init() {
+	cmdMsgBroker.Run = runMsgBroker // break init cycle
+	messageBrokerStandaloneOptions.filer = cmdMsgBroker.Flag.String("filer", "localhost:8888", "filer server address")
+	messageBrokerStandaloneOptions.port = cmdMsgBroker.Flag.Int("port", 17777, "queue server gRPC listen port")
+	messageBrokerStandaloneOptions.defaultTtl = cmdMsgBroker.Flag.String("ttl", "1h", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
+}
+
+var cmdMsgBroker = &Command{
+	UsageLine: "msg.broker [-port=17777] [-filer=<ip:port>]",
+	Short:     "<WIP> start a message queue broker",
+	Long: `start a message queue broker
+
+	The broker can accept gRPC calls to write or read messages. The messages are stored via filer.
+	The brokers are stateless. To scale up, just add more brokers.
+
+`,
+}
+
+func runMsgBroker(cmd *Command, args []string) bool {
+
+	util.LoadConfiguration("security", false)
+
+	return messageBrokerStandaloneOptions.startQueueServer()
+
+}
+
+func (msgBrokerOpt *QueueOptions) startQueueServer() bool {
+
+	filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*msgBrokerOpt.filer)
+	if err != nil {
+		glog.Fatal(err)
+		return false
+	}
+
+	filerQueuesPath := "/queues"
+
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	for {
+		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			if err != nil {
+				return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
+			}
+			filerQueuesPath = resp.DirQueues
+			glog.V(0).Infof("Queue read filer queues dir: %s", filerQueuesPath)
+			return nil
+		})
+		if err != nil {
+			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *msgBrokerOpt.filer, filerGrpcAddress)
+			time.Sleep(time.Second)
+		} else {
+			glog.V(0).Infof("connected to filer %s grpc address %s", *msgBrokerOpt.filer, filerGrpcAddress)
+			break
+		}
+	}
+
+	qs, err := weed_server.NewMessageBroker(&weed_server.MessageBrokerOption{
+		Filers:             []string{*msgBrokerOpt.filer},
+		DefaultReplication: "",
+		MaxMB:              0,
+		Port:               *msgBrokerOpt.port,
+	})
+
+	// start grpc listener
+	grpcL, err := util.NewListener(":"+strconv.Itoa(*msgBrokerOpt.port), 0)
+	if err != nil {
+		glog.Fatalf("failed to listen on grpc port %d: %v", *msgBrokerOpt.port, err)
+	}
+	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.msg_broker"))
+	queue_pb.RegisterSeaweedQueueServer(grpcS, qs)
+	reflection.Register(grpcS)
+	grpcS.Serve(grpcL)
+
+	return true
+
+}

weed/command/s3.go

@@ -1,18 +1,20 @@
 package command
 
 import (
+	"context"
+	"fmt"
 	"net/http"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/spf13/viper"
 
-	"fmt"
+	"github.com/gorilla/mux"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/s3api"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/gorilla/mux"
 )
 
 var (
@@ -20,29 +22,89 @@ var (
 )
 
 type S3Options struct {
-	filer            *string
-	filerBucketsPath *string
-	port             *int
-	domainName       *string
-	tlsPrivateKey    *string
-	tlsCertificate   *string
+	filer          *string
+	port           *int
+	config         *string
+	domainName     *string
+	tlsPrivateKey  *string
+	tlsCertificate *string
 }
 
 func init() {
 	cmdS3.Run = runS3 // break init cycle
 	s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
-	s3StandaloneOptions.filerBucketsPath = cmdS3.Flag.String("filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
 	s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
 	s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name, {bucket}.{domainName}")
+	s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file")
 	s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
 	s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
 }
 
 var cmdS3 = &Command{
-	UsageLine: "s3 -port=8333 -filer=<ip:port>",
+	UsageLine: "s3 [-port=8333] [-filer=<ip:port>] [-config=</path/to/config.json>]",
 	Short:     "start a s3 API compatible server that is backed by a filer",
 	Long: `start a s3 API compatible server that is backed by a filer.
 
+	By default, you can use any access key and secret key to access the S3 APIs.
+	To enable credential based access, create a config.json file similar to this:
+
+{
+  "identities": [
+    {
+      "name": "some_name",
+      "credentials": [
+        {
+          "accessKey": "some_access_key1",
+          "secretKey": "some_secret_key1"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "Write"
+      ]
+    },
+    {
+      "name": "some_read_only_user",
+      "credentials": [
+        {
+          "accessKey": "some_access_key2",
+          "secretKey": "some_secret_key2"
+        }
+      ],
+      "actions": [
+        "Read"
+      ]
+    },
+    {
+      "name": "some_normal_user",
+      "credentials": [
+        {
+          "accessKey": "some_access_key3",
+          "secretKey": "some_secret_key3"
+        }
+      ],
+      "actions": [
+        "Read",
+        "Write"
+      ]
+    },
+    {
+      "name": "user_limited_to_bucket1",
+      "credentials": [
+        {
+          "accessKey": "some_access_key4",
+          "secretKey": "some_secret_key4"
+        }
+      ],
+      "actions": [
+        "Read:bucket1",
+        "Write:bucket1"
+      ]
+    }
+  ]
+}
+
 `,
 }
 
@@ -56,20 +118,44 @@ func runS3(cmd *Command, args []string) bool {
 
 func (s3opt *S3Options) startS3Server() bool {
 
-	filerGrpcAddress, err := parseFilerGrpcAddress(*s3opt.filer)
+	filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*s3opt.filer)
 	if err != nil {
 		glog.Fatal(err)
 		return false
 	}
 
+	filerBucketsPath := "/buckets"
+
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	for {
+		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			if err != nil {
+				return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
+			}
+			filerBucketsPath = resp.DirBuckets
+			glog.V(0).Infof("S3 read filer buckets dir: %s", filerBucketsPath)
+			return nil
+		})
+		if err != nil {
+			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *s3opt.filer, filerGrpcAddress)
+			time.Sleep(time.Second)
+		} else {
+			glog.V(0).Infof("connected to filer %s grpc address %s", *s3opt.filer, filerGrpcAddress)
+			break
+		}
+	}
+
 	router := mux.NewRouter().SkipClean(true)
 
 	_, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
 		Filer:            *s3opt.filer,
 		FilerGrpcAddress: filerGrpcAddress,
+		Config:           *s3opt.config,
 		DomainName:       *s3opt.domainName,
-		BucketsPath:      *s3opt.filerBucketsPath,
-		GrpcDialOption:   security.LoadClientTLS(viper.Sub("grpc"), "client"),
+		BucketsPath:      filerBucketsPath,
+		GrpcDialOption:   grpcDialOption,
 	})
 	if s3ApiServer_err != nil {
 		glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)

weed/command/scaffold.go

@@ -14,6 +14,14 @@ var cmdScaffold = &Command{
 	Short:     "generate basic configuration files",
 	Long: `Generate filer.toml with all possible configurations for you to customize.
 
+	The options can also be overwritten by environment variables.
+	For example, the filer.toml mysql password can be overwritten by environment variable
+		export WEED_MYSQL_PASSWORD=some_password
+	Environment variable rules:
+		* Prefix fix with "WEED_"
+		* Upppercase the reset of variable name.
+		* Replace '.' with '_'
+
   `,
 }
 
@@ -59,16 +67,28 @@ const (
 #    $HOME/.seaweedfs/filer.toml
 #    /etc/seaweedfs/filer.toml
 
+####################################################
+# Customizable filer server options
+####################################################
+[filer.options]
+# with http DELETE, by default the filer would check whether a folder is empty.
+# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
+recursive_delete = false
+# directories under this folder will be automatically creating a separate bucket
+buckets_folder = "/buckets"
+# directories under this folder will be store message queue data
+queues_folder = "/queues"
+
+####################################################
+# The following are filer store options
+####################################################
+
 [leveldb2]
 # local on disk, mostly for simple single-machine setup, fairly scalable
 # faster than previous leveldb, recommended.
 enabled = true
 dir = "."					# directory to store level db files
 
-####################################################
-# multiple filers on shared storage, fairly scalable
-####################################################
-
 [mysql]  # or tidb
 # CREATE TABLE IF NOT EXISTS filemeta (
 #   dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
@@ -136,6 +156,10 @@ addresses = [
     "localhost:30006",
 ]
 password = ""
+# allows reads from slave servers or the master, but all writes still go to the master
+readOnly = true
+# automatically use the closest Redis server for reads
+routeByLatency = true
 
 [etcd]
 enabled = false
@@ -302,6 +326,10 @@ key  = ""
 cert = ""
 key  = ""
 
+[grpc.msg_broker]
+cert = ""
+key  = ""
+
 # use this for any place needs a grpc client
 # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
 [grpc.client]
@@ -348,6 +376,8 @@ type = "memory"     # Choose [memory|etcd] type for storing the file id sequence
 sequencer_etcd_urls = "http://127.0.0.1:2379"
 
 
+# configurations for tiered cloud storage
+# old volumes are transparently moved to cloud for cost efficiency
 [storage.backend]
 	[storage.backend.s3.default]
 	enabled = false
@@ -356,5 +386,17 @@ sequencer_etcd_urls = "http://127.0.0.1:2379"
 	region = "us-east-2"
 	bucket = "your_bucket_name"    # an existing bucket
 
+# create this number of logical volumes if no more writable volumes
+# count_x means how many copies of data.
+# e.g.:
+#   000 has only one copy, count_1
+#   010 and 001 has two copies, count_2
+#   011 has only 3 copies, count_3
+[master.volume_growth]
+count_1 = 7                # create 1 x 7 = 7 actual volumes
+count_2 = 6                # create 2 x 6 = 12 actual volumes
+count_3 = 3                # create 3 x 3 = 9 actual volumes
+count_other = 1            # create n x 1 = n actual volumes
+
 `
 )

weed/command/server.go

@@ -78,10 +78,10 @@ func init() {
 	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
 	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
 	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
-	filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
 	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
 	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
 	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
+	filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
@@ -89,13 +89,14 @@ func init() {
 	serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
 	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
 	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
+	serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory")
 	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
 
-	s3Options.filerBucketsPath = cmdServer.Flag.String("s3.filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
 	s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
 	s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}")
 	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
 	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
+	s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
 
 }
 
@@ -113,10 +114,6 @@ func runServer(cmd *Command, args []string) bool {
 		defer pprof.StopCPUProfile()
 	}
 
-	if *filerOptions.redirectOnRead {
-		*isStartingFiler = true
-	}
-
 	if *isStartingS3 {
 		*isStartingFiler = true
 	}

weed/command/shell.go

@@ -6,7 +6,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/shell"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/spf13/viper"
 )
 
 var (
@@ -31,7 +30,7 @@ var cmdShell = &Command{
 func runShell(command *Command, args []string) bool {
 
 	util.LoadConfiguration("security", false)
-	shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
+	shellOptions.GrpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	var filerPwdErr error
 	shellOptions.FilerHost, shellOptions.FilerPort, shellOptions.Directory, filerPwdErr = util.ParseFilerUrl(*shellInitialFilerUrl)

weed/command/upload.go

@@ -6,11 +6,9 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/spf13/viper"
-
-	"github.com/chrislusf/seaweedfs/weed/operation"
 )
 
 var (
@@ -63,7 +61,7 @@ var cmdUpload = &Command{
 func runUpload(cmd *Command, args []string) bool {
 
 	util.LoadConfiguration("security", false)
-	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	if len(args) == 0 {
 		if *upload.dir == "" {

weed/command/volume.go

@@ -10,17 +10,20 @@ import (
 	"strings"
 	"time"
 
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/util/httpdown"
 	"github.com/spf13/viper"
 	"google.golang.org/grpc"
 
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util/httpdown"
+
+	"google.golang.org/grpc/reflection"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"google.golang.org/grpc/reflection"
 )
 
 var (
@@ -47,6 +50,7 @@ type VolumeServerOptions struct {
 	cpuProfile            *string
 	memProfile            *string
 	compactionMBPerSecond *int
+	fileSizeLimitMB       *int
 }
 
 func init() {
@@ -67,6 +71,7 @@ func init() {
 	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
 	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
 	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
+	v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory")
 }
 
 var cmdVolume = &Command{
@@ -158,6 +163,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		v.whiteList,
 		*v.fixJpgOrientation, *v.readRedirect,
 		*v.compactionMBPerSecond,
+		*v.fileSizeLimitMB,
 	)
 
 	// starting grpc server
@@ -229,7 +235,7 @@ func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerSe
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
 	}
-	grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "volume"))
+	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.volume"))
 	volume_server_pb.RegisterVolumeServerServer(grpcS, vs)
 	reflection.Register(grpcS)
 	go func() {

weed/command/webdav.go

@@ -1,6 +1,7 @@
 package command
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"os/user"
@@ -8,10 +9,11 @@ import (
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/spf13/viper"
 )
 
 var (
@@ -37,7 +39,7 @@ func init() {
 
 var cmdWebDav = &Command{
 	UsageLine: "webdav -port=7333 -filer=<ip:port>",
-	Short:     "<unstable> start a webdav server that is backed by a filer",
+	Short:     "start a webdav server that is backed by a filer",
 	Long: `start a webdav server that is backed by a filer.
 
 `,
@@ -55,12 +57,6 @@ func runWebDav(cmd *Command, args []string) bool {
 
 func (wo *WebDavOption) startWebDav() bool {
 
-	filerGrpcAddress, err := parseFilerGrpcAddress(*wo.filer)
-	if err != nil {
-		glog.Fatal(err)
-		return false
-	}
-
 	// detect current user
 	uid, gid := uint32(0), uint32(0)
 	if u, err := user.Current(); err == nil {
@@ -72,13 +68,43 @@ func (wo *WebDavOption) startWebDav() bool {
 		}
 	}
 
+	// parse filer grpc address
+	filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*wo.filer)
+	if err != nil {
+		glog.Fatal(err)
+		return false
+	}
+
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	var cipher bool
+	// connect to filer
+	for {
+		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			if err != nil {
+				return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
+			}
+			cipher = resp.Cipher
+			return nil
+		})
+		if err != nil {
+			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *wo.filer, filerGrpcAddress)
+			time.Sleep(time.Second)
+		} else {
+			glog.V(0).Infof("connected to filer %s grpc address %s", *wo.filer, filerGrpcAddress)
+			break
+		}
+	}
+
 	ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{
 		Filer:            *wo.filer,
 		FilerGrpcAddress: filerGrpcAddress,
-		GrpcDialOption:   security.LoadClientTLS(viper.Sub("grpc"), "client"),
+		GrpcDialOption:   grpcDialOption,
 		Collection:       *wo.collection,
 		Uid:              uid,
 		Gid:              gid,
+		Cipher:           cipher,
 	})
 	if webdavServer_err != nil {
 		glog.Fatalf("WebDav Server startup error: %v", webdavServer_err)

weed/filer2/abstract_sql/abstract_sql_store.go

@@ -7,6 +7,8 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 type AbstractSqlStore struct {
@@ -65,7 +67,7 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer2.En
 		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
 	}
 
-	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, hashToLong(dir), name, dir, meta)
+	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, util.HashStringToLong(dir), name, dir, meta)
 	if err != nil {
 		return fmt.Errorf("insert %s: %s", entry.FullPath, err)
 	}
@@ -85,7 +87,7 @@ func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer2.En
 		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
 	}
 
-	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, hashToLong(dir), name, dir)
+	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, util.HashStringToLong(dir), name, dir)
 	if err != nil {
 		return fmt.Errorf("update %s: %s", entry.FullPath, err)
 	}
@@ -100,10 +102,10 @@ func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer2.En
 func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (*filer2.Entry, error) {
 
 	dir, name := fullpath.DirAndName()
-	row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, hashToLong(dir), name, dir)
+	row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, util.HashStringToLong(dir), name, dir)
 	var data []byte
 	if err := row.Scan(&data); err != nil {
-		return nil, filer2.ErrNotFound
+		return nil, filer_pb.ErrNotFound
 	}
 
 	entry := &filer2.Entry{
@@ -120,7 +122,7 @@ func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath filer2.
 
 	dir, name := fullpath.DirAndName()
 
-	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, hashToLong(dir), name, dir)
+	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, util.HashStringToLong(dir), name, dir)
 	if err != nil {
 		return fmt.Errorf("delete %s: %s", fullpath, err)
 	}
@@ -135,7 +137,7 @@ func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath filer2.
 
 func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpath filer2.FullPath) error {
 
-	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDeleteFolderChildren, hashToLong(string(fullpath)), fullpath)
+	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDeleteFolderChildren, util.HashStringToLong(string(fullpath)), fullpath)
 	if err != nil {
 		return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err)
 	}
@@ -155,7 +157,7 @@ func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpat
 		sqlText = store.SqlListInclusive
 	}
 
-	rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, hashToLong(string(fullpath)), startFileName, string(fullpath), limit)
+	rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, util.HashStringToLong(string(fullpath)), startFileName, string(fullpath), limit)
 	if err != nil {
 		return nil, fmt.Errorf("list %s : %v", fullpath, err)
 	}

weed/filer2/abstract_sql/hashing.go

@@ -1,32 +0,0 @@
-package abstract_sql
-
-import (
-	"crypto/md5"
-	"io"
-)
-
-// returns a 64 bit big int
-func hashToLong(dir string) (v int64) {
-	h := md5.New()
-	io.WriteString(h, dir)
-
-	b := h.Sum(nil)
-
-	v += int64(b[0])
-	v <<= 8
-	v += int64(b[1])
-	v <<= 8
-	v += int64(b[2])
-	v <<= 8
-	v += int64(b[3])
-	v <<= 8
-	v += int64(b[4])
-	v <<= 8
-	v += int64(b[5])
-	v <<= 8
-	v += int64(b[6])
-	v <<= 8
-	v += int64(b[7])
-
-	return
-}

weed/filer2/cassandra/cassandra_store.go

@@ -3,10 +3,13 @@ package cassandra
 import (
 	"context"
 	"fmt"
+
+	"github.com/gocql/gocql"
+
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/gocql/gocql"
 )
 
 func init() {
@@ -22,10 +25,10 @@ func (store *CassandraStore) GetName() string {
 	return "cassandra"
 }
 
-func (store *CassandraStore) Initialize(configuration util.Configuration) (err error) {
+func (store *CassandraStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
-		configuration.GetString("keyspace"),
-		configuration.GetStringSlice("hosts"),
+		configuration.GetString(prefix+"keyspace"),
+		configuration.GetStringSlice(prefix+"hosts"),
 	)
 }
 
@@ -80,12 +83,12 @@ func (store *CassandraStore) FindEntry(ctx context.Context, fullpath filer2.Full
 		"SELECT meta FROM filemeta WHERE directory=? AND name=?",
 		dir, name).Consistency(gocql.One).Scan(&data); err != nil {
 		if err != gocql.ErrNotFound {
-			return nil, filer2.ErrNotFound
+			return nil, filer_pb.ErrNotFound
 		}
 	}
 
 	if len(data) == 0 {
-		return nil, filer2.ErrNotFound
+		return nil, filer_pb.ErrNotFound
 	}
 
 	entry = &filer2.Entry{

weed/filer2/configuration.go

@@ -17,8 +17,7 @@ func (f *Filer) LoadConfiguration(config *viper.Viper) {
 
 	for _, store := range Stores {
 		if config.GetBool(store.GetName() + ".enabled") {
-			viperSub := config.Sub(store.GetName())
-			if err := store.Initialize(viperSub); err != nil {
+			if err := store.Initialize(config, store.GetName()+"."); err != nil {
 				glog.Fatalf("Failed to initialize store for %s: %+v",
 					store.GetName(), err)
 			}

weed/filer2/entry.go

@@ -30,6 +30,7 @@ type Entry struct {
 	FullPath
 
 	Attr
+	Extended map[string][]byte
 
 	// the following is for files
 	Chunks []*filer_pb.FileChunk `json:"chunks,omitempty"`
@@ -56,6 +57,7 @@ func (entry *Entry) ToProtoEntry() *filer_pb.Entry {
 		IsDirectory: entry.IsDirectory(),
 		Attributes:  EntryAttributeToPb(entry),
 		Chunks:      entry.Chunks,
+		Extended:    entry.Extended,
 	}
 }
 

weed/filer2/entry_codec.go

@@ -1,18 +1,21 @@
 package filer2
 
 import (
+	"bytes"
+	"fmt"
 	"os"
 	"time"
 
-	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/golang/protobuf/proto"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
 
 func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {
 	message := &filer_pb.Entry{
 		Attributes: EntryAttributeToPb(entry),
 		Chunks:     entry.Chunks,
+		Extended:   entry.Extended,
 	}
 	return proto.Marshal(message)
 }
@@ -27,6 +30,8 @@ func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error {
 
 	entry.Attr = PbToEntryAttribute(message.Attributes)
 
+	entry.Extended = message.Extended
+
 	entry.Chunks = message.Chunks
 
 	return nil
@@ -84,6 +89,10 @@ func EqualEntry(a, b *Entry) bool {
 		return false
 	}
 
+	if !eq(a.Extended, b.Extended) {
+		return false
+	}
+
 	for i := 0; i < len(a.Chunks); i++ {
 		if !proto.Equal(a.Chunks[i], b.Chunks[i]) {
 			return false
@@ -91,3 +100,17 @@ func EqualEntry(a, b *Entry) bool {
 	}
 	return true
 }
+
+func eq(a, b map[string][]byte) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for k, v := range a {
+		if w, ok := b[k]; !ok || !bytes.Equal(v, w) {
+			return false
+		}
+	}
+
+	return true
+}

weed/filer2/etcd/etcd_store.go

@@ -6,10 +6,12 @@ import (
 	"strings"
 	"time"
 
+	"go.etcd.io/etcd/clientv3"
+
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	weed_util "github.com/chrislusf/seaweedfs/weed/util"
-	"go.etcd.io/etcd/clientv3"
 )
 
 const (
@@ -28,13 +30,13 @@ func (store *EtcdStore) GetName() string {
 	return "etcd"
 }
 
-func (store *EtcdStore) Initialize(configuration weed_util.Configuration) (err error) {
-	servers := configuration.GetString("servers")
+func (store *EtcdStore) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
+	servers := configuration.GetString(prefix + "servers")
 	if servers == "" {
 		servers = "localhost:2379"
 	}
 
-	timeout := configuration.GetString("timeout")
+	timeout := configuration.GetString(prefix + "timeout")
 	if timeout == "" {
 		timeout = "3s"
 	}
@@ -99,7 +101,7 @@ func (store *EtcdStore) FindEntry(ctx context.Context, fullpath filer2.FullPath)
 	}
 
 	if len(resp.Kvs) == 0 {
-		return nil, filer2.ErrNotFound
+		return nil, filer_pb.ErrNotFound
 	}
 
 	entry = &filer2.Entry{

weed/filer2/filechunks.go

@@ -71,6 +71,8 @@ type ChunkView struct {
 	Size        uint64
 	LogicOffset int64
 	IsFullChunk bool
+	CipherKey   []byte
+	isGzipped   bool
 }
 
 func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) {
@@ -86,6 +88,7 @@ func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int
 	stop := offset + int64(size)
 
 	for _, chunk := range visibles {
+
 		if chunk.start <= offset && offset < chunk.stop && offset < stop {
 			isFullChunk := chunk.isFullChunk && chunk.start == offset && chunk.stop <= stop
 			views = append(views, &ChunkView{
@@ -94,6 +97,8 @@ func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int
 				Size:        uint64(min(chunk.stop, stop) - offset),
 				LogicOffset: offset,
 				IsFullChunk: isFullChunk,
+				CipherKey:   chunk.cipherKey,
+				isGzipped:   chunk.isGzipped,
 			})
 			offset = min(chunk.stop, stop)
 		}
@@ -120,13 +125,7 @@ var bufPool = sync.Pool{
 
 func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.FileChunk) []VisibleInterval {
 
-	newV := newVisibleInterval(
-		chunk.Offset,
-		chunk.Offset+int64(chunk.Size),
-		chunk.GetFileIdString(),
-		chunk.Mtime,
-		true,
-	)
+	newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, true, chunk.CipherKey, chunk.IsGzipped)
 
 	length := len(visibles)
 	if length == 0 {
@@ -140,23 +139,11 @@ func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.
 	logPrintf("  before", visibles)
 	for _, v := range visibles {
 		if v.start < chunk.Offset && chunk.Offset < v.stop {
-			newVisibles = append(newVisibles, newVisibleInterval(
-				v.start,
-				chunk.Offset,
-				v.fileId,
-				v.modifiedTime,
-				false,
-			))
+			newVisibles = append(newVisibles, newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTime, false, v.cipherKey, v.isGzipped))
 		}
 		chunkStop := chunk.Offset + int64(chunk.Size)
 		if v.start < chunkStop && chunkStop < v.stop {
-			newVisibles = append(newVisibles, newVisibleInterval(
-				chunkStop,
-				v.stop,
-				v.fileId,
-				v.modifiedTime,
-				false,
-			))
+			newVisibles = append(newVisibles, newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTime, false, v.cipherKey, v.isGzipped))
 		}
 		if chunkStop <= v.start || v.stop <= chunk.Offset {
 			newVisibles = append(newVisibles, v)
@@ -187,6 +174,7 @@ func NonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []Vi
 
 	var newVisibles []VisibleInterval
 	for _, chunk := range chunks {
+
 		newVisibles = MergeIntoVisibles(visibles, newVisibles, chunk)
 		t := visibles[:0]
 		visibles = newVisibles
@@ -208,15 +196,19 @@ type VisibleInterval struct {
 	modifiedTime int64
 	fileId       string
 	isFullChunk  bool
+	cipherKey    []byte
+	isGzipped    bool
 }
 
-func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool) VisibleInterval {
+func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool, cipherKey []byte, isGzipped bool) VisibleInterval {
 	return VisibleInterval{
 		start:        start,
 		stop:         stop,
 		fileId:       fileId,
 		modifiedTime: modifiedTime,
 		isFullChunk:  isFullChunk,
+		cipherKey:    cipherKey,
+		isGzipped:    isGzipped,
 	}
 }
 

weed/filer2/filechunks_test.go

@@ -331,6 +331,42 @@ func TestChunksReading(t *testing.T) {
 				{Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 100},
 			},
 		},
+		// case 8: edge cases
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 90, Size: 200, FileId: "asdf", Mtime: 134},
+				{Offset: 190, Size: 300, FileId: "fsad", Mtime: 353},
+			},
+			Offset: 0,
+			Size:   300,
+			Expected: []*ChunkView{
+				{Offset: 0, Size: 90, FileId: "abc", LogicOffset: 0},
+				{Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 90},
+				{Offset: 0, Size: 110, FileId: "fsad", LogicOffset: 190},
+			},
+		},
+		// case 9: edge cases
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 43175947, FileId: "2,111fc2cbfac1", Mtime: 1},
+				{Offset: 43175936, Size: 52981771 - 43175936, FileId: "2,112a36ea7f85", Mtime: 2},
+				{Offset: 52981760, Size: 72564747 - 52981760, FileId: "4,112d5f31c5e7", Mtime: 3},
+				{Offset: 72564736, Size: 133255179 - 72564736, FileId: "1,113245f0cdb6", Mtime: 4},
+				{Offset: 133255168, Size: 137269259 - 133255168, FileId: "3,1141a70733b5", Mtime: 5},
+				{Offset: 137269248, Size: 153578836 - 137269248, FileId: "1,114201d5bbdb", Mtime: 6},
+			},
+			Offset: 0,
+			Size:   153578836,
+			Expected: []*ChunkView{
+				{Offset: 0, Size: 43175936, FileId: "2,111fc2cbfac1", LogicOffset: 0},
+				{Offset: 0, Size: 52981760 - 43175936, FileId: "2,112a36ea7f85", LogicOffset: 43175936},
+				{Offset: 0, Size: 72564736 - 52981760, FileId: "4,112d5f31c5e7", LogicOffset: 52981760},
+				{Offset: 0, Size: 133255168 - 72564736, FileId: "1,113245f0cdb6", LogicOffset: 72564736},
+				{Offset: 0, Size: 137269248 - 133255168, FileId: "3,1141a70733b5", LogicOffset: 133255168},
+				{Offset: 0, Size: 153578836 - 137269248, FileId: "1,114201d5bbdb", LogicOffset: 137269248},
+			},
+		},
 	}
 
 	for i, testcase := range testcases {

weed/filer2/filer.go

@@ -13,6 +13,8 @@ import (
 	"github.com/karlseguin/ccache"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )
 
@@ -24,19 +26,23 @@ var (
 )
 
 type Filer struct {
-	store              *FilerStoreWrapper
-	directoryCache     *ccache.Cache
-	MasterClient       *wdclient.MasterClient
-	fileIdDeletionChan chan string
-	GrpcDialOption     grpc.DialOption
+	store               *FilerStoreWrapper
+	directoryCache      *ccache.Cache
+	MasterClient        *wdclient.MasterClient
+	fileIdDeletionQueue *util.UnboundedQueue
+	GrpcDialOption      grpc.DialOption
+	DirBucketsPath      string
+	DirQueuesPath       string
+	buckets             *FilerBuckets
+	Cipher              bool
 }
 
-func NewFiler(masters []string, grpcDialOption grpc.DialOption) *Filer {
+func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort uint32) *Filer {
 	f := &Filer{
-		directoryCache:     ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
-		MasterClient:       wdclient.NewMasterClient(context.Background(), grpcDialOption, "filer", masters),
-		fileIdDeletionChan: make(chan string, PaginationSize),
-		GrpcDialOption:     grpcDialOption,
+		directoryCache:      ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
+		MasterClient:        wdclient.NewMasterClient(grpcDialOption, "filer", filerGrpcPort, masters),
+		fileIdDeletionQueue: util.NewUnboundedQueue(),
+		GrpcDialOption:      grpcDialOption,
 	}
 
 	go f.loopProcessingDeletion()
@@ -72,7 +78,7 @@ func (f *Filer) RollbackTransaction(ctx context.Context) error {
 	return f.store.RollbackTransaction(ctx)
 }
 
-func (f *Filer) CreateEntry(ctx context.Context, entry *Entry) error {
+func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool) error {
 
 	if string(entry.FullPath) == "/" {
 		return nil
@@ -96,7 +102,7 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry) error {
 			glog.V(4).Infof("find uncached directory: %s", dirPath)
 			dirEntry, _ = f.FindEntry(ctx, FullPath(dirPath))
 		} else {
-			glog.V(4).Infof("found cached directory: %s", dirPath)
+			// glog.V(4).Infof("found cached directory: %s", dirPath)
 		}
 
 		// no such existing directory
@@ -108,25 +114,30 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry) error {
 			dirEntry = &Entry{
 				FullPath: FullPath(dirPath),
 				Attr: Attr{
-					Mtime:  now,
-					Crtime: now,
-					Mode:   os.ModeDir | 0770,
-					Uid:    entry.Uid,
-					Gid:    entry.Gid,
+					Mtime:       now,
+					Crtime:      now,
+					Mode:        os.ModeDir | 0770,
+					Uid:         entry.Uid,
+					Gid:         entry.Gid,
+					Collection:  entry.Collection,
+					Replication: entry.Replication,
 				},
 			}
 
 			glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
 			mkdirErr := f.store.InsertEntry(ctx, dirEntry)
 			if mkdirErr != nil {
-				if _, err := f.FindEntry(ctx, FullPath(dirPath)); err == ErrNotFound {
+				if _, err := f.FindEntry(ctx, FullPath(dirPath)); err == filer_pb.ErrNotFound {
+					glog.V(3).Infof("mkdir %s: %v", dirPath, mkdirErr)
 					return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
 				}
 			} else {
+				f.maybeAddBucket(dirEntry)
 				f.NotifyUpdateEvent(nil, dirEntry, false)
 			}
 
 		} else if !dirEntry.IsDirectory() {
+			glog.Errorf("CreateEntry %s: %s should be a directory", entry.FullPath, dirPath)
 			return fmt.Errorf("%s is a file", dirPath)
 		}
 
@@ -141,6 +152,7 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry) error {
 	}
 
 	if lastDirectoryEntry == nil {
+		glog.Errorf("CreateEntry %s: lastDirectoryEntry is nil", entry.FullPath)
 		return fmt.Errorf("parent folder not found: %v", entry.FullPath)
 	}
 
@@ -154,22 +166,30 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry) error {
 
 	oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
 
+	glog.V(4).Infof("CreateEntry %s: old entry: %v exclusive:%v", entry.FullPath, oldEntry, o_excl)
 	if oldEntry == nil {
 		if err := f.store.InsertEntry(ctx, entry); err != nil {
 			glog.Errorf("insert entry %s: %v", entry.FullPath, err)
 			return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
 		}
 	} else {
+		if o_excl {
+			glog.V(3).Infof("EEXIST: entry %s already exists", entry.FullPath)
+			return fmt.Errorf("EEXIST: entry %s already exists", entry.FullPath)
+		}
 		if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
 			glog.Errorf("update entry %s: %v", entry.FullPath, err)
 			return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
 		}
 	}
 
+	f.maybeAddBucket(entry)
 	f.NotifyUpdateEvent(oldEntry, entry, true)
 
 	f.deleteChunksIfNotNew(oldEntry, entry)
 
+	glog.V(4).Infof("CreateEntry %s: created", entry.FullPath)
+
 	return nil
 }
 
@@ -203,14 +223,51 @@ func (f *Filer) FindEntry(ctx context.Context, p FullPath) (entry *Entry, err er
 			},
 		}, nil
 	}
-	return f.store.FindEntry(ctx, p)
+	entry, err = f.store.FindEntry(ctx, p)
+	if entry != nil && entry.TtlSec > 0 {
+		if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
+			f.store.DeleteEntry(ctx, p.Child(entry.Name()))
+			return nil, filer_pb.ErrNotFound
+		}
+	}
+	return
+
 }
 
 func (f *Filer) ListDirectoryEntries(ctx context.Context, p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
 	if strings.HasSuffix(string(p), "/") && len(p) > 1 {
 		p = p[0 : len(p)-1]
 	}
-	return f.store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit)
+
+	var makeupEntries []*Entry
+	entries, expiredCount, lastFileName, err := f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit)
+	for expiredCount > 0 && err == nil {
+		makeupEntries, expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount)
+		if err == nil {
+			entries = append(entries, makeupEntries...)
+		}
+	}
+
+	return entries, err
+}
+
+func (f *Filer) doListDirectoryEntries(ctx context.Context, p FullPath, startFileName string, inclusive bool, limit int) (entries []*Entry, expiredCount int, lastFileName string, err error) {
+	listedEntries, listErr := f.store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit)
+	if listErr != nil {
+		return listedEntries, expiredCount, "", listErr
+	}
+	for _, entry := range listedEntries {
+		lastFileName = entry.Name()
+		if entry.TtlSec > 0 {
+			if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
+				f.store.DeleteEntry(ctx, p.Child(entry.Name()))
+				expiredCount++
+				continue
+			}
+		}
+		entries = append(entries, entry)
+	}
+	return
 }
 
 func (f *Filer) cacheDelDirectory(dirpath string) {

weed/filer2/filer_buckets.go

@@ -0,0 +1,113 @@
+package filer2
+
+import (
+	"context"
+	"math"
+	"sync"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+)
+
+type BucketName string
+type BucketOption struct {
+	Name        BucketName
+	Replication string
+}
+type FilerBuckets struct {
+	dirBucketsPath string
+	buckets        map[BucketName]*BucketOption
+	sync.RWMutex
+}
+
+func (f *Filer) LoadBuckets(dirBucketsPath string) {
+
+	f.buckets = &FilerBuckets{
+		buckets: make(map[BucketName]*BucketOption),
+	}
+	f.DirBucketsPath = dirBucketsPath
+
+	limit := math.MaxInt32
+
+	entries, err := f.ListDirectoryEntries(context.Background(), FullPath(dirBucketsPath), "", false, limit)
+
+	if err != nil {
+		glog.V(1).Infof("no buckets found: %v", err)
+		return
+	}
+
+	glog.V(1).Infof("buckets found: %d", len(entries))
+
+	f.buckets.Lock()
+	for _, entry := range entries {
+		f.buckets.buckets[BucketName(entry.Name())] = &BucketOption{
+			Name:        BucketName(entry.Name()),
+			Replication: entry.Replication,
+		}
+	}
+	f.buckets.Unlock()
+
+}
+
+func (f *Filer) ReadBucketOption(buketName string) (replication string) {
+
+	f.buckets.RLock()
+	defer f.buckets.RUnlock()
+
+	option, found := f.buckets.buckets[BucketName(buketName)]
+
+	if !found {
+		return ""
+	}
+	return option.Replication
+
+}
+
+func (f *Filer) isBucket(entry *Entry) bool {
+	if !entry.IsDirectory() {
+		return false
+	}
+	parent, dirName := entry.FullPath.DirAndName()
+	if parent != f.DirBucketsPath {
+		return false
+	}
+
+	f.buckets.RLock()
+	defer f.buckets.RUnlock()
+
+	_, found := f.buckets.buckets[BucketName(dirName)]
+
+	return found
+
+}
+
+func (f *Filer) maybeAddBucket(entry *Entry) {
+	if !entry.IsDirectory() {
+		return
+	}
+	parent, dirName := entry.FullPath.DirAndName()
+	if parent != f.DirBucketsPath {
+		return
+	}
+	f.addBucket(dirName, &BucketOption{
+		Name:        BucketName(dirName),
+		Replication: entry.Replication,
+	})
+}
+
+func (f *Filer) addBucket(buketName string, bucketOption *BucketOption) {
+
+	f.buckets.Lock()
+	defer f.buckets.Unlock()
+
+	f.buckets.buckets[BucketName(buketName)] = bucketOption
+
+}
+
+func (f *Filer) deleteBucket(buketName string) {
+
+	f.buckets.Lock()
+	defer f.buckets.Unlock()
+
+	delete(f.buckets.buckets, BucketName(buketName))
+
+}

weed/filer2/filer_client_util.go

@@ -22,10 +22,11 @@ func VolumeId(fileId string) string {
 }
 
 type FilerClient interface {
-	WithFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error
+	WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error
+	AdjustedUrl(hostAndPort string) string
 }
 
-func ReadIntoBuffer(ctx context.Context, filerClient FilerClient, fullFilePath string, buff []byte, chunkViews []*ChunkView, baseOffset int64) (totalRead int64, err error) {
+func ReadIntoBuffer(filerClient FilerClient, fullFilePath FullPath, buff []byte, chunkViews []*ChunkView, baseOffset int64) (totalRead int64, err error) {
 	var vids []string
 	for _, chunkView := range chunkViews {
 		vids = append(vids, VolumeId(chunkView.FileId))
@@ -33,10 +34,10 @@ func ReadIntoBuffer(ctx context.Context, filerClient FilerClient, fullFilePath s
 
 	vid2Locations := make(map[string]*filer_pb.Locations)
 
-	err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		glog.V(4).Infof("read fh lookup volume id locations: %v", vids)
-		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
+		resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{
 			VolumeIds: vids,
 		})
 		if err != nil {
@@ -67,20 +68,16 @@ func ReadIntoBuffer(ctx context.Context, filerClient FilerClient, fullFilePath s
 				return
 			}
 
+			volumeServerAddress := filerClient.AdjustedUrl(locations.Locations[0].Url)
 			var n int64
-			n, err = util.ReadUrl(
-				fmt.Sprintf("http://%s/%s", locations.Locations[0].Url, chunkView.FileId),
-				chunkView.Offset,
-				int(chunkView.Size),
-				buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)],
-				!chunkView.IsFullChunk)
+			n, err = util.ReadUrl(fmt.Sprintf("http://%s/%s", volumeServerAddress, chunkView.FileId), chunkView.CipherKey, chunkView.isGzipped, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)])
 
 			if err != nil {
 
-				glog.V(0).Infof("%v read http://%s/%v %v bytes: %v", fullFilePath, locations.Locations[0].Url, chunkView.FileId, n, err)
+				glog.V(0).Infof("%v read http://%s/%v %v bytes: %v", fullFilePath, volumeServerAddress, chunkView.FileId, n, err)
 
 				err = fmt.Errorf("failed to read http://%s/%s: %v",
-					locations.Locations[0].Url, chunkView.FileId, err)
+					volumeServerAddress, chunkView.FileId, err)
 				return
 			}
 
@@ -93,29 +90,29 @@ func ReadIntoBuffer(ctx context.Context, filerClient FilerClient, fullFilePath s
 	return
 }
 
-func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath string) (entry *filer_pb.Entry, err error) {
+func GetEntry(filerClient FilerClient, fullFilePath FullPath) (entry *filer_pb.Entry, err error) {
 
-	dir, name := FullPath(fullFilePath).DirAndName()
+	dir, name := fullFilePath.DirAndName()
 
-	err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Directory: dir,
 			Name:      name,
 		}
 
-		glog.V(3).Infof("read %s request: %v", fullFilePath, request)
-		resp, err := client.LookupDirectoryEntry(ctx, request)
+		// glog.V(3).Infof("read %s request: %v", fullFilePath, request)
+		resp, err := filer_pb.LookupEntry(client, request)
 		if err != nil {
-			if err == ErrNotFound || strings.Contains(err.Error(), ErrNotFound.Error()) {
+			if err == filer_pb.ErrNotFound {
 				return nil
 			}
-			glog.V(3).Infof("read %s attr %v: %v", fullFilePath, request, err)
+			glog.V(3).Infof("read %s %v: %v", fullFilePath, resp, err)
 			return err
 		}
 
 		if resp.Entry == nil {
-			glog.V(3).Infof("read %s entry: %v", fullFilePath, entry)
+			// glog.V(3).Infof("read %s entry: %v", fullFilePath, entry)
 			return nil
 		}
 
@@ -126,21 +123,21 @@ func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath string)
 	return
 }
 
-func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath, prefix string, fn func(entry *filer_pb.Entry, isLast bool)) (err error) {
+func ReadDirAllEntries(filerClient FilerClient, fullDirPath FullPath, prefix string, fn func(entry *filer_pb.Entry, isLast bool)) (err error) {
 
-	err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		lastEntryName := ""
 
 		request := &filer_pb.ListEntriesRequest{
-			Directory:         fullDirPath,
+			Directory:         string(fullDirPath),
 			Prefix:            prefix,
 			StartFromFileName: lastEntryName,
 			Limit:             math.MaxUint32,
 		}
 
 		glog.V(3).Infof("read directory: %v", request)
-		stream, err := client.ListEntries(ctx, request)
+		stream, err := client.ListEntries(context.Background(), request)
 		if err != nil {
 			return fmt.Errorf("list %s: %v", fullDirPath, err)
 		}

weed/filer2/filer_delete_entry.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 )
 
 func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecursive bool, ignoreRecursiveError, shouldDeleteChunks bool) (err error) {
@@ -18,27 +19,35 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecurs
 		return findErr
 	}
 
+	isCollection := f.isBucket(entry)
+
 	var chunks []*filer_pb.FileChunk
 	chunks = append(chunks, entry.Chunks...)
 	if entry.IsDirectory() {
 		// delete the folder children, not including the folder itself
 		var dirChunks []*filer_pb.FileChunk
-		dirChunks, err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks)
+		dirChunks, err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isCollection)
 		if err != nil {
+			glog.V(0).Infof("delete directory %s: %v", p, err)
 			return fmt.Errorf("delete directory %s: %v", p, err)
 		}
 		chunks = append(chunks, dirChunks...)
-		f.cacheDelDirectory(string(p))
 	}
+
 	// delete the file or folder
 	err = f.doDeleteEntryMetaAndData(ctx, entry, shouldDeleteChunks)
 	if err != nil {
 		return fmt.Errorf("delete file %s: %v", p, err)
 	}
 
-	if shouldDeleteChunks {
+	if shouldDeleteChunks && !isCollection {
 		go f.DeleteChunks(chunks)
 	}
+	if isCollection {
+		collectionName := entry.Name()
+		f.doDeleteCollection(collectionName)
+		f.deleteBucket(collectionName)
+	}
 
 	return nil
 }
@@ -63,13 +72,13 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 			var dirChunks []*filer_pb.FileChunk
 			if sub.IsDirectory() {
 				dirChunks, err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks)
+				chunks = append(chunks, dirChunks...)
+			} else {
+				chunks = append(chunks, sub.Chunks...)
 			}
 			if err != nil && !ignoreRecursiveError {
 				return nil, err
 			}
-			if shouldDeleteChunks {
-				chunks = append(chunks, dirChunks...)
-			}
 		}
 
 		if len(entries) < PaginationSize {
@@ -79,7 +88,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 
 	f.cacheDelDirectory(string(entry.FullPath))
 
-	glog.V(3).Infof("deleting directory %v", entry.FullPath)
+	glog.V(3).Infof("deleting directory %v delete %d chunks: %v", entry.FullPath, len(chunks), shouldDeleteChunks)
 
 	if storeDeletionErr := f.store.DeleteFolderChildren(ctx, entry.FullPath); storeDeletionErr != nil {
 		return nil, fmt.Errorf("filer store delete: %v", storeDeletionErr)
@@ -91,7 +100,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 
 func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shouldDeleteChunks bool) (err error) {
 
-	glog.V(3).Infof("deleting entry %v", entry.FullPath)
+	glog.V(3).Infof("deleting entry %v, delete chunks: %v", entry.FullPath, shouldDeleteChunks)
 
 	if storeDeletionErr := f.store.DeleteEntry(ctx, entry.FullPath); storeDeletionErr != nil {
 		return fmt.Errorf("filer store delete: %v", storeDeletionErr)
@@ -100,3 +109,17 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou
 
 	return nil
 }
+
+func (f *Filer) doDeleteCollection(collectionName string) (err error) {
+
+	return f.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
+		_, err := client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{
+			Name: collectionName,
+		})
+		if err != nil {
+			glog.Infof("delete collection %s: %v", collectionName, err)
+		}
+		return err
+	})
+
+}

weed/filer2/filer_deletion.go

@@ -10,8 +10,6 @@ import (
 
 func (f *Filer) loopProcessingDeletion() {
 
-	ticker := time.NewTicker(5 * time.Second)
-
 	lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) {
 		m := make(map[string]operation.LookupResult)
 		for _, vid := range vids {
@@ -31,36 +29,35 @@ func (f *Filer) loopProcessingDeletion() {
 		return m, nil
 	}
 
-	var fileIds []string
+	var deletionCount int
 	for {
-		select {
-		case fid := <-f.fileIdDeletionChan:
-			fileIds = append(fileIds, fid)
-			if len(fileIds) >= 4096 {
-				glog.V(1).Infof("deleting fileIds len=%d", len(fileIds))
-				operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, fileIds, lookupFunc)
-				fileIds = fileIds[:0]
-			}
-		case <-ticker.C:
-			if len(fileIds) > 0 {
-				glog.V(1).Infof("timed deletion fileIds len=%d", len(fileIds))
-				operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, fileIds, lookupFunc)
-				fileIds = fileIds[:0]
+		deletionCount = 0
+		f.fileIdDeletionQueue.Consume(func(fileIds []string) {
+			deletionCount = len(fileIds)
+			_, err := operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, fileIds, lookupFunc)
+			if err != nil {
+				glog.V(0).Infof("deleting fileIds len=%d error: %v", deletionCount, err)
+			} else {
+				glog.V(1).Infof("deleting fileIds len=%d", deletionCount)
 			}
+		})
+
+		if deletionCount == 0 {
+			time.Sleep(1123 * time.Millisecond)
 		}
 	}
 }
 
 func (f *Filer) DeleteChunks(chunks []*filer_pb.FileChunk) {
 	for _, chunk := range chunks {
-		f.fileIdDeletionChan <- chunk.GetFileIdString()
+		f.fileIdDeletionQueue.EnQueue(chunk.GetFileIdString())
 	}
 }
 
 // DeleteFileByFileId direct delete by file id.
 // Only used when the fileId is not being managed by snapshots.
 func (f *Filer) DeleteFileByFileId(fileId string) {
-	f.fileIdDeletionChan <- fileId
+	f.fileIdDeletionQueue.EnQueue(fileId)
 }
 
 func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {

weed/filer2/filerstore.go

@@ -2,7 +2,6 @@ package filer2
 
 import (
 	"context"
-	"errors"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@@ -14,7 +13,7 @@ type FilerStore interface {
 	// GetName gets the name to locate the configuration in filer.toml file
 	GetName() string
 	// Initialize initializes the file store
-	Initialize(configuration util.Configuration) error
+	Initialize(configuration util.Configuration, prefix string) error
 	InsertEntry(context.Context, *Entry) error
 	UpdateEntry(context.Context, *Entry) (err error)
 	// err == filer2.ErrNotFound if not found
@@ -28,8 +27,6 @@ type FilerStore interface {
 	RollbackTransaction(ctx context.Context) error
 }
 
-var ErrNotFound = errors.New("filer: no entry is found in filer store")
-
 type FilerStoreWrapper struct {
 	actualStore FilerStore
 }
@@ -47,8 +44,8 @@ func (fsw *FilerStoreWrapper) GetName() string {
 	return fsw.actualStore.GetName()
 }
 
-func (fsw *FilerStoreWrapper) Initialize(configuration util.Configuration) error {
-	return fsw.actualStore.Initialize(configuration)
+func (fsw *FilerStoreWrapper) Initialize(configuration util.Configuration, prefix string) error {
+	return fsw.actualStore.Initialize(configuration, prefix)
 }
 
 func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) error {

weed/filer2/fullpath.go

@@ -3,6 +3,8 @@ package filer2
 import (
 	"path/filepath"
 	"strings"
+
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 type FullPath string
@@ -34,3 +36,7 @@ func (fp FullPath) Child(name string) FullPath {
 	}
 	return FullPath(dir + "/" + name)
 }
+
+func (fp FullPath) AsInode() uint64 {
+	return uint64(util.HashStringToLong(string(fp)))
+}

weed/filer2/leveldb/leveldb_store.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	weed_util "github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -30,8 +31,8 @@ func (store *LevelDBStore) GetName() string {
 	return "leveldb"
 }
 
-func (store *LevelDBStore) Initialize(configuration weed_util.Configuration) (err error) {
-	dir := configuration.GetString("dir")
+func (store *LevelDBStore) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
+	dir := configuration.GetString(prefix + "dir")
 	return store.initialize(dir)
 }
 
@@ -94,7 +95,7 @@ func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath filer2.FullPa
 	data, err := store.db.Get(key, nil)
 
 	if err == leveldb.ErrNotFound {
-		return nil, filer2.ErrNotFound
+		return nil, filer_pb.ErrNotFound
 	}
 	if err != nil {
 		return nil, fmt.Errorf("get %s : %v", entry.FullPath, err)

weed/filer2/leveldb/leveldb_store_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	filer := filer2.NewFiler(nil, nil)
+	filer := filer2.NewFiler(nil, nil, 0)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
@@ -30,7 +30,7 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := filer.CreateEntry(ctx, entry1); err != nil {
+	if err := filer.CreateEntry(ctx, entry1, false); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}
@@ -64,7 +64,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	filer := filer2.NewFiler(nil, nil)
+	filer := filer2.NewFiler(nil, nil, 0)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}

weed/filer2/leveldb2/leveldb2_store.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	weed_util "github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -30,8 +31,8 @@ func (store *LevelDB2Store) GetName() string {
 	return "leveldb2"
 }
 
-func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration) (err error) {
-	dir := configuration.GetString("dir")
+func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
+	dir := configuration.GetString(prefix + "dir")
 	return store.initialize(dir, 8)
 }
 
@@ -104,7 +105,7 @@ func (store *LevelDB2Store) FindEntry(ctx context.Context, fullpath filer2.FullP
 	data, err := store.dbs[partitionId].Get(key, nil)
 
 	if err == leveldb.ErrNotFound {
-		return nil, filer2.ErrNotFound
+		return nil, filer_pb.ErrNotFound
 	}
 	if err != nil {
 		return nil, fmt.Errorf("get %s : %v", entry.FullPath, err)

weed/filer2/leveldb2/leveldb2_store_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	filer := filer2.NewFiler(nil, nil)
+	filer := filer2.NewFiler(nil, nil, 0)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDB2Store{}
@@ -30,7 +30,7 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := filer.CreateEntry(ctx, entry1); err != nil {
+	if err := filer.CreateEntry(ctx, entry1, false); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}
@@ -64,7 +64,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	filer := filer2.NewFiler(nil, nil)
+	filer := filer2.NewFiler(nil, nil, 0)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDB2Store{}

weed/filer2/mysql/mysql_store.go

@@ -26,16 +26,16 @@ func (store *MysqlStore) GetName() string {
 	return "mysql"
 }
 
-func (store *MysqlStore) Initialize(configuration util.Configuration) (err error) {
+func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
-		configuration.GetString("username"),
-		configuration.GetString("password"),
-		configuration.GetString("hostname"),
-		configuration.GetInt("port"),
-		configuration.GetString("database"),
-		configuration.GetInt("connection_max_idle"),
-		configuration.GetInt("connection_max_open"),
-		configuration.GetBool("interpolateParams"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetString(prefix+"hostname"),
+		configuration.GetInt(prefix+"port"),
+		configuration.GetString(prefix+"database"),
+		configuration.GetInt(prefix+"connection_max_idle"),
+		configuration.GetInt(prefix+"connection_max_open"),
+		configuration.GetBool(prefix+"interpolateParams"),
 	)
 }
 

weed/filer2/postgres/postgres_store.go

@@ -11,7 +11,7 @@ import (
 )
 
 const (
-	CONNECTION_URL_PATTERN = "host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30"
+	CONNECTION_URL_PATTERN = "host=%s port=%d user=%s sslmode=%s connect_timeout=30"
 )
 
 func init() {
@@ -26,16 +26,16 @@ func (store *PostgresStore) GetName() string {
 	return "postgres"
 }
 
-func (store *PostgresStore) Initialize(configuration util.Configuration) (err error) {
+func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
-		configuration.GetString("username"),
-		configuration.GetString("password"),
-		configuration.GetString("hostname"),
-		configuration.GetInt("port"),
-		configuration.GetString("database"),
-		configuration.GetString("sslmode"),
-		configuration.GetInt("connection_max_idle"),
-		configuration.GetInt("connection_max_open"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetString(prefix+"hostname"),
+		configuration.GetInt(prefix+"port"),
+		configuration.GetString(prefix+"database"),
+		configuration.GetString(prefix+"sslmode"),
+		configuration.GetInt(prefix+"connection_max_idle"),
+		configuration.GetInt(prefix+"connection_max_open"),
 	)
 }
 
@@ -49,7 +49,13 @@ func (store *PostgresStore) initialize(user, password, hostname string, port int
 	store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>$2 AND directory=$3 ORDER BY NAME ASC LIMIT $4"
 	store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 ORDER BY NAME ASC LIMIT $4"
 
-	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, user, password, database, sslmode)
+	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, user, sslmode)
+	if password != "" {
+		sqlUrl += " password="+password
+	}
+	if database != "" {
+		sqlUrl += " dbname="+database
+	}
 	var dbErr error
 	store.DB, dbErr = sql.Open("postgres", sqlUrl)
 	if dbErr != nil {

weed/filer2/redis/redis_cluster_store.go

@@ -18,17 +18,25 @@ func (store *RedisClusterStore) GetName() string {
 	return "redis_cluster"
 }
 
-func (store *RedisClusterStore) Initialize(configuration util.Configuration) (err error) {
+func (store *RedisClusterStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+
+	configuration.SetDefault(prefix+"useReadOnly", true)
+	configuration.SetDefault(prefix+"routeByLatency", true)
+
 	return store.initialize(
-		configuration.GetStringSlice("addresses"),
-		configuration.GetString("password"),
+		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetBool(prefix+"useReadOnly"),
+		configuration.GetBool(prefix+"routeByLatency"),
 	)
 }
 
-func (store *RedisClusterStore) initialize(addresses []string, password string) (err error) {
+func (store *RedisClusterStore) initialize(addresses []string, password string, readOnly, routeByLatency bool) (err error) {
 	store.Client = redis.NewClusterClient(&redis.ClusterOptions{
-		Addrs:    addresses,
-		Password: password,
+		Addrs:          addresses,
+		Password:       password,
+		ReadOnly:       readOnly,
+		RouteByLatency: routeByLatency,
 	})
 	return
 }

weed/filer2/redis/redis_store.go

@@ -18,11 +18,11 @@ func (store *RedisStore) GetName() string {
 	return "redis"
 }
 
-func (store *RedisStore) Initialize(configuration util.Configuration) (err error) {
+func (store *RedisStore) Initialize(configuration util.Configuration, prefix string) (err error) {
 	return store.initialize(
-		configuration.GetString("address"),
-		configuration.GetString("password"),
-		configuration.GetInt("database"),
+		configuration.GetString(prefix+"address"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetInt(prefix+"database"),
 	)
 }
 

weed/filer2/redis/universal_redis_store.go

@@ -3,12 +3,15 @@ package redis
 import (
 	"context"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/filer2"
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/go-redis/redis"
 	"sort"
 	"strings"
 	"time"
+
+	"github.com/go-redis/redis"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
 
 const (
@@ -62,7 +65,7 @@ func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath filer2
 
 	data, err := store.Client.Get(string(fullpath)).Result()
 	if err == redis.Nil {
-		return nil, filer2.ErrNotFound
+		return nil, filer_pb.ErrNotFound
 	}
 
 	if err != nil {
@@ -120,7 +123,8 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full
 func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
 	limit int) (entries []*filer2.Entry, err error) {
 
-	members, err := store.Client.SMembers(genDirectoryListKey(string(fullpath))).Result()
+	dirListKey := genDirectoryListKey(string(fullpath))
+	members, err := store.Client.SMembers(dirListKey).Result()
 	if err != nil {
 		return nil, fmt.Errorf("list %s : %v", fullpath, err)
 	}
@@ -159,6 +163,13 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full
 		if err != nil {
 			glog.V(0).Infof("list %s : %v", path, err)
 		} else {
+			if entry.TtlSec > 0 {
+				if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
+					store.Client.Del(string(path)).Result()
+					store.Client.SRem(dirListKey, fileName).Result()
+					continue
+				}
+			}
 			entries = append(entries, entry)
 		}
 	}

weed/filer2/stream.go

@@ -26,8 +26,9 @@ func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*f
 	}
 
 	for _, chunkView := range chunkViews {
+
 		urlString := fileId2Url[chunkView.FileId]
-		_, err := util.ReadUrlAsStream(urlString, chunkView.Offset, int(chunkView.Size), func(data []byte) {
+		err := util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.isGzipped, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), func(data []byte) {
 			w.Write(data)
 		})
 		if err != nil {

weed/filer2/tikv/tikv_store.go

@@ -1,5 +1,6 @@
 // +build !386
 // +build !arm
+// +build !windows
 
 package tikv
 
@@ -12,6 +13,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	weed_util "github.com/chrislusf/seaweedfs/weed/util"
 
 	"github.com/pingcap/tidb/kv"
@@ -30,8 +32,8 @@ func (store *TikvStore) GetName() string {
 	return "tikv"
 }
 
-func (store *TikvStore) Initialize(configuration weed_util.Configuration) (err error) {
-	pdAddr := configuration.GetString("pdAddress")
+func (store *TikvStore) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
+	pdAddr := configuration.GetString(prefix + "pdAddress")
 	return store.initialize(pdAddr)
 }
 
@@ -110,7 +112,7 @@ func (store *TikvStore) FindEntry(ctx context.Context, fullpath filer2.FullPath)
 	data, err := store.getTx(ctx).Get(ctx, key)
 
 	if err == kv.ErrNotExist {
-		return nil, filer2.ErrNotFound
+		return nil, filer_pb.ErrNotFound
 	}
 	if err != nil {
 		return nil, fmt.Errorf("get %s : %v", entry.FullPath, err)

weed/filer2/tikv/tikv_store_unsupported.go

@@ -1,4 +1,4 @@
-// +build 386 arm
+// +build 386 arm windows
 
 package tikv
 
@@ -21,7 +21,7 @@ func (store *TikvStore) GetName() string {
 	return "tikv"
 }
 
-func (store *TikvStore) Initialize(configuration weed_util.Configuration) (err error) {
+func (store *TikvStore) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
 	return fmt.Errorf("not implemented for 32 bit computers")
 }
 

weed/filesys/dir.go

@@ -3,7 +3,7 @@ package filesys
 import (
 	"context"
 	"os"
-	"path"
+	"strings"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
@@ -14,9 +14,9 @@ import (
 )
 
 type Dir struct {
-	Path       string
-	wfs        *WFS
-	entry      *filer_pb.Entry
+	Path  string
+	wfs   *WFS
+	entry *filer_pb.Entry
 }
 
 var _ = fs.Node(&Dir{})
@@ -31,29 +31,33 @@ var _ = fs.NodeGetxattrer(&Dir{})
 var _ = fs.NodeSetxattrer(&Dir{})
 var _ = fs.NodeRemovexattrer(&Dir{})
 var _ = fs.NodeListxattrer(&Dir{})
+var _ = fs.NodeForgetter(&Dir{})
 
 func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
 
-	glog.V(3).Infof("dir Attr %s", dir.Path)
-
 	// https://github.com/bazil/fuse/issues/196
 	attr.Valid = time.Second
 
 	if dir.Path == dir.wfs.option.FilerMountRootPath {
 		dir.setRootDirAttributes(attr)
+		glog.V(3).Infof("root dir Attr %s, attr: %+v", dir.Path, attr)
 		return nil
 	}
 
-	if err := dir.maybeLoadEntry(ctx); err != nil {
+	if err := dir.maybeLoadEntry(); err != nil {
+		glog.V(3).Infof("dir Attr %s,err: %+v", dir.Path, err)
 		return err
 	}
 
+	attr.Inode = filer2.FullPath(dir.Path).AsInode()
 	attr.Mode = os.FileMode(dir.entry.Attributes.FileMode) | os.ModeDir
 	attr.Mtime = time.Unix(dir.entry.Attributes.Mtime, 0)
-	attr.Ctime = time.Unix(dir.entry.Attributes.Crtime, 0)
+	attr.Crtime = time.Unix(dir.entry.Attributes.Crtime, 0)
 	attr.Gid = dir.entry.Attributes.Gid
 	attr.Uid = dir.entry.Attributes.Uid
 
+	glog.V(3).Infof("dir Attr %s, attr: %+v", dir.Path, attr)
+
 	return nil
 }
 
@@ -61,7 +65,7 @@ func (dir *Dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *f
 
 	glog.V(4).Infof("dir Getxattr %s", dir.Path)
 
-	if err := dir.maybeLoadEntry(ctx); err != nil {
+	if err := dir.maybeLoadEntry(); err != nil {
 		return err
 	}
 
@@ -69,6 +73,8 @@ func (dir *Dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *f
 }
 
 func (dir *Dir) setRootDirAttributes(attr *fuse.Attr) {
+	attr.Inode = 1 // filer2.FullPath(dir.Path).AsInode()
+	attr.Valid = time.Hour
 	attr.Uid = dir.wfs.option.MountUid
 	attr.Gid = dir.wfs.option.MountGid
 	attr.Mode = dir.wfs.option.MountMode
@@ -76,16 +82,25 @@ func (dir *Dir) setRootDirAttributes(attr *fuse.Attr) {
 	attr.Ctime = dir.wfs.option.MountCtime
 	attr.Mtime = dir.wfs.option.MountMtime
 	attr.Atime = dir.wfs.option.MountMtime
+	attr.BlockSize = 1024 * 1024
 }
 
-func (dir *Dir) newFile(name string, entry *filer_pb.Entry) *File {
-	return &File{
-		Name:           name,
-		dir:            dir,
-		wfs:            dir.wfs,
-		entry:          entry,
-		entryViewCache: nil,
-	}
+func (dir *Dir) newFile(name string, entry *filer_pb.Entry) fs.Node {
+	return dir.wfs.getNode(filer2.NewFullPath(dir.Path, name), func() fs.Node {
+		return &File{
+			Name:           name,
+			dir:            dir,
+			wfs:            dir.wfs,
+			entry:          entry,
+			entryViewCache: nil,
+		}
+	})
+}
+
+func (dir *Dir) newDirectory(fullpath filer2.FullPath, entry *filer_pb.Entry) fs.Node {
+	return dir.wfs.getNode(fullpath, func() fs.Node {
+		return &Dir{Path: string(fullpath), wfs: dir.wfs, entry: entry}
+	})
 }
 
 func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
@@ -107,102 +122,104 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 				TtlSec:      dir.wfs.option.TtlSec,
 			},
 		},
+		OExcl: req.Flags&fuse.OpenExclusive != 0,
 	}
-	glog.V(1).Infof("create: %v", request)
+	glog.V(1).Infof("create %s/%s: %v", dir.Path, req.Name, req.Flags)
 
-	if request.Entry.IsDirectory {
-		if err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
-			if _, err := client.CreateEntry(ctx, request); err != nil {
-				glog.V(0).Infof("create %s/%s: %v", dir.Path, req.Name, err)
-				return fuse.EIO
+	if err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		if err := filer_pb.CreateEntry(client, request); err != nil {
+			if strings.Contains(err.Error(), "EEXIST") {
+				return fuse.EEXIST
 			}
-			return nil
-		}); err != nil {
-			return nil, nil, err
+			return fuse.EIO
 		}
+		return nil
+	}); err != nil {
+		return nil, nil, err
+	}
+	var node fs.Node
+	if request.Entry.IsDirectory {
+		node = dir.newDirectory(filer2.NewFullPath(dir.Path, req.Name), request.Entry)
+		return node, nil, nil
 	}
 
-	file := dir.newFile(req.Name, request.Entry)
-	if !request.Entry.IsDirectory {
-		file.isOpen = true
-	}
+	node = dir.newFile(req.Name, request.Entry)
+	file := node.(*File)
+	file.isOpen++
 	fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid)
-	fh.dirtyMetadata = true
 	return file, fh, nil
 
 }
 
 func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
 
-	err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	newEntry := &filer_pb.Entry{
+		Name:        req.Name,
+		IsDirectory: true,
+		Attributes: &filer_pb.FuseAttributes{
+			Mtime:    time.Now().Unix(),
+			Crtime:   time.Now().Unix(),
+			FileMode: uint32(req.Mode &^ dir.wfs.option.Umask),
+			Uid:      req.Uid,
+			Gid:      req.Gid,
+		},
+	}
+
+	err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.CreateEntryRequest{
 			Directory: dir.Path,
-			Entry: &filer_pb.Entry{
-				Name:        req.Name,
-				IsDirectory: true,
-				Attributes: &filer_pb.FuseAttributes{
-					Mtime:    time.Now().Unix(),
-					Crtime:   time.Now().Unix(),
-					FileMode: uint32(req.Mode &^ dir.wfs.option.Umask),
-					Uid:      req.Uid,
-					Gid:      req.Gid,
-				},
-			},
+			Entry:     newEntry,
 		}
 
 		glog.V(1).Infof("mkdir: %v", request)
-		if _, err := client.CreateEntry(ctx, request); err != nil {
+		if err := filer_pb.CreateEntry(client, request); err != nil {
 			glog.V(0).Infof("mkdir %s/%s: %v", dir.Path, req.Name, err)
-			return fuse.EIO
+			return err
 		}
 
 		return nil
 	})
 
 	if err == nil {
-		node := &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs}
+		node := dir.newDirectory(filer2.NewFullPath(dir.Path, req.Name), newEntry)
 		return node, nil
 	}
 
-	return nil, err
+	return nil, fuse.EIO
 }
 
 func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fs.Node, err error) {
 
 	glog.V(4).Infof("dir Lookup %s: %s", dir.Path, req.Name)
 
-	var entry *filer_pb.Entry
-	fullFilePath := path.Join(dir.Path, req.Name)
-
-	item := dir.wfs.listDirectoryEntriesCache.Get(fullFilePath)
-	if item != nil && !item.Expired() {
-		entry = item.Value().(*filer_pb.Entry)
-	}
+	fullFilePath := filer2.NewFullPath(dir.Path, req.Name)
+	entry := dir.wfs.cacheGet(fullFilePath)
 
 	if entry == nil {
-		glog.V(3).Infof("dir Lookup cache miss %s", fullFilePath)
-		entry, err = filer2.GetEntry(ctx, dir.wfs, fullFilePath)
+		// glog.V(3).Infof("dir Lookup cache miss %s", fullFilePath)
+		entry, err = filer2.GetEntry(dir.wfs, fullFilePath)
 		if err != nil {
-			return nil, err
-		}
-		if entry != nil {
-			dir.wfs.listDirectoryEntriesCache.Set(fullFilePath, entry, 5*time.Minute)
+			glog.V(1).Infof("dir GetEntry %s: %v", fullFilePath, err)
+			return nil, fuse.ENOENT
 		}
+		dir.wfs.cacheSet(fullFilePath, entry, 5*time.Minute)
 	} else {
 		glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath)
 	}
 
 	if entry != nil {
 		if entry.IsDirectory {
-			node = &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs, entry: entry}
+			node = dir.newDirectory(fullFilePath, entry)
 		} else {
 			node = dir.newFile(req.Name, entry)
 		}
 
-		resp.EntryValid = time.Duration(0)
+		// resp.EntryValid = time.Second
+		resp.Attr.Inode = fullFilePath.AsInode()
+		resp.Attr.Valid = time.Second
 		resp.Attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
-		resp.Attr.Ctime = time.Unix(entry.Attributes.Crtime, 0)
+		resp.Attr.Crtime = time.Unix(entry.Attributes.Crtime, 0)
 		resp.Attr.Mode = os.FileMode(entry.Attributes.FileMode)
 		resp.Attr.Gid = entry.Attributes.Gid
 		resp.Attr.Uid = entry.Attributes.Uid
@@ -210,6 +227,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
 		return node, nil
 	}
 
+	glog.V(4).Infof("not found dir GetEntry %s: %v", fullFilePath, err)
 	return nil, fuse.ENOENT
 }
 
@@ -219,15 +237,17 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 
 	cacheTtl := 5 * time.Minute
 
-	readErr := filer2.ReadDirAllEntries(ctx, dir.wfs, dir.Path, "", func(entry *filer_pb.Entry, isLast bool) {
+	readErr := filer2.ReadDirAllEntries(dir.wfs, filer2.FullPath(dir.Path), "", func(entry *filer_pb.Entry, isLast bool) {
+		fullpath := filer2.NewFullPath(dir.Path, entry.Name)
+		inode := fullpath.AsInode()
 		if entry.IsDirectory {
-			dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
+			dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: fuse.DT_Dir}
 			ret = append(ret, dirent)
 		} else {
-			dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
+			dirent := fuse.Dirent{Inode: inode, Name: entry.Name, Type: fuse.DT_File}
 			ret = append(ret, dirent)
 		}
-		dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, cacheTtl)
+		dir.wfs.cacheSet(fullpath, entry, cacheTtl)
 	})
 	if readErr != nil {
 		glog.V(0).Infof("list %s: %v", dir.Path, err)
@@ -240,25 +260,29 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
 
 	if !req.Dir {
-		return dir.removeOneFile(ctx, req)
+		return dir.removeOneFile(req)
 	}
 
-	return dir.removeFolder(ctx, req)
+	return dir.removeFolder(req)
 
 }
 
-func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) error {
+func (dir *Dir) removeOneFile(req *fuse.RemoveRequest) error {
 
-	entry, err := filer2.GetEntry(ctx, dir.wfs, path.Join(dir.Path, req.Name))
+	filePath := filer2.NewFullPath(dir.Path, req.Name)
+	entry, err := filer2.GetEntry(dir.wfs, filePath)
 	if err != nil {
 		return err
 	}
+	if entry == nil {
+		return nil
+	}
 
-	dir.wfs.deleteFileChunks(ctx, entry.Chunks)
+	dir.wfs.deleteFileChunks(entry.Chunks)
 
-	dir.wfs.listDirectoryEntriesCache.Delete(path.Join(dir.Path, req.Name))
+	dir.wfs.cacheDelete(filePath)
 
-	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    dir.Path,
@@ -267,9 +291,9 @@ func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) erro
 		}
 
 		glog.V(3).Infof("remove file: %v", request)
-		_, err := client.DeleteEntry(ctx, request)
+		_, err := client.DeleteEntry(context.Background(), request)
 		if err != nil {
-			glog.V(3).Infof("remove file %s/%s: %v", dir.Path, req.Name, err)
+			glog.V(3).Infof("not found remove file %s/%s: %v", dir.Path, req.Name, err)
 			return fuse.ENOENT
 		}
 
@@ -278,11 +302,11 @@ func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) erro
 
 }
 
-func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error {
+func (dir *Dir) removeFolder(req *fuse.RemoveRequest) error {
 
-	dir.wfs.listDirectoryEntriesCache.Delete(path.Join(dir.Path, req.Name))
+	dir.wfs.cacheDelete(filer2.NewFullPath(dir.Path, req.Name))
 
-	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    dir.Path,
@@ -291,9 +315,9 @@ func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error
 		}
 
 		glog.V(3).Infof("remove directory entry: %v", request)
-		_, err := client.DeleteEntry(ctx, request)
+		_, err := client.DeleteEntry(context.Background(), request)
 		if err != nil {
-			glog.V(3).Infof("remove %s/%s: %v", dir.Path, req.Name, err)
+			glog.V(3).Infof("not found remove %s/%s: %v", dir.Path, req.Name, err)
 			return fuse.ENOENT
 		}
 
@@ -304,11 +328,12 @@ func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error
 
 func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
 
-	if err := dir.maybeLoadEntry(ctx); err != nil {
+	glog.V(3).Infof("%v dir setattr %+v", dir.Path, req)
+
+	if err := dir.maybeLoadEntry(); err != nil {
 		return err
 	}
 
-	glog.V(3).Infof("%v dir setattr %+v, fh=%d", dir.Path, req, req.Handle)
 	if req.Valid.Mode() {
 		dir.entry.Attributes.FileMode = uint32(req.Mode)
 	}
@@ -325,9 +350,9 @@ func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus
 		dir.entry.Attributes.Mtime = req.Mtime.Unix()
 	}
 
-	dir.wfs.listDirectoryEntriesCache.Delete(dir.Path)
+	dir.wfs.cacheDelete(filer2.FullPath(dir.Path))
 
-	return dir.saveEntry(ctx)
+	return dir.saveEntry()
 
 }
 
@@ -335,7 +360,7 @@ func (dir *Dir) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {
 
 	glog.V(4).Infof("dir Setxattr %s: %s", dir.Path, req.Name)
 
-	if err := dir.maybeLoadEntry(ctx); err != nil {
+	if err := dir.maybeLoadEntry(); err != nil {
 		return err
 	}
 
@@ -343,7 +368,9 @@ func (dir *Dir) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {
 		return err
 	}
 
-	return dir.saveEntry(ctx)
+	dir.wfs.cacheDelete(filer2.FullPath(dir.Path))
+
+	return dir.saveEntry()
 
 }
 
@@ -351,7 +378,7 @@ func (dir *Dir) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) e
 
 	glog.V(4).Infof("dir Removexattr %s: %s", dir.Path, req.Name)
 
-	if err := dir.maybeLoadEntry(ctx); err != nil {
+	if err := dir.maybeLoadEntry(); err != nil {
 		return err
 	}
 
@@ -359,7 +386,9 @@ func (dir *Dir) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) e
 		return err
 	}
 
-	return dir.saveEntry(ctx)
+	dir.wfs.cacheDelete(filer2.FullPath(dir.Path))
+
+	return dir.saveEntry()
 
 }
 
@@ -367,7 +396,7 @@ func (dir *Dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp
 
 	glog.V(4).Infof("dir Listxattr %s", dir.Path)
 
-	if err := dir.maybeLoadEntry(ctx); err != nil {
+	if err := dir.maybeLoadEntry(); err != nil {
 		return err
 	}
 
@@ -375,30 +404,33 @@ func (dir *Dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp
 		return err
 	}
 
-	return dir.saveEntry(ctx)
+	return nil
 
 }
 
-func (dir *Dir) maybeLoadEntry(ctx context.Context) error {
+func (dir *Dir) Forget() {
+	glog.V(3).Infof("Forget dir %s", dir.Path)
+
+	dir.wfs.forgetNode(filer2.FullPath(dir.Path))
+}
+
+func (dir *Dir) maybeLoadEntry() error {
 	if dir.entry == nil {
 		parentDirPath, name := filer2.FullPath(dir.Path).DirAndName()
-		entry, err := dir.wfs.maybeLoadEntry(ctx, parentDirPath, name)
+		entry, err := dir.wfs.maybeLoadEntry(parentDirPath, name)
 		if err != nil {
 			return err
 		}
-		if entry == nil {
-			return fuse.ENOENT
-		}
 		dir.entry = entry
 	}
 	return nil
 }
 
-func (dir *Dir) saveEntry(ctx context.Context) error {
+func (dir *Dir) saveEntry() error {
 
 	parentDir, name := filer2.FullPath(dir.Path).DirAndName()
 
-	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.UpdateEntryRequest{
 			Directory: parentDir,
@@ -406,7 +438,7 @@ func (dir *Dir) saveEntry(ctx context.Context) error {
 		}
 
 		glog.V(1).Infof("save dir entry: %v", request)
-		_, err := client.UpdateEntry(ctx, request)
+		_, err := client.UpdateEntry(context.Background(), request)
 		if err != nil {
 			glog.V(0).Infof("UpdateEntry dir %s/%s: %v", parentDir, name, err)
 			return fuse.EIO

weed/filesys/dir_link.go

@@ -35,8 +35,8 @@ func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node,
 		},
 	}
 
-	err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
-		if _, err := client.CreateEntry(ctx, request); err != nil {
+	err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		if err := filer_pb.CreateEntry(client, request); err != nil {
 			glog.V(0).Infof("symlink %s/%s: %v", dir.Path, req.NewName, err)
 			return fuse.EIO
 		}

weed/filesys/dir_rename.go

@@ -2,7 +2,9 @@ package filesys
 
 import (
 	"context"
-	"fmt"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/fuse"
 	"github.com/seaweedfs/fuse/fs"
@@ -11,8 +13,9 @@ import (
 func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {
 
 	newDir := newDirectory.(*Dir)
+	glog.V(4).Infof("dir Rename %s/%s => %s/%s", dir.Path, req.OldName, newDir.Path, req.NewName)
 
-	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.AtomicRenameEntryRequest{
 			OldDirectory: dir.Path,
@@ -21,13 +24,40 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 			NewName:      req.NewName,
 		}
 
-		_, err := client.AtomicRenameEntry(ctx, request)
+		_, err := client.AtomicRenameEntry(context.Background(), request)
 		if err != nil {
-			return fmt.Errorf("renaming %s/%s => %s/%s: %v", dir.Path, req.OldName, newDir.Path, req.NewName, err)
+			glog.V(0).Infof("dir Rename %s/%s => %s/%s : %v", dir.Path, req.OldName, newDir.Path, req.NewName, err)
+			return fuse.EIO
 		}
 
 		return nil
 
 	})
 
+	if err == nil {
+		newPath := filer2.NewFullPath(newDir.Path, req.NewName)
+		oldPath := filer2.NewFullPath(dir.Path, req.OldName)
+		dir.wfs.cacheDelete(newPath)
+		dir.wfs.cacheDelete(oldPath)
+
+		oldFileNode := dir.wfs.getNode(oldPath, func() fs.Node {
+			return nil
+		})
+		newDirNode := dir.wfs.getNode(filer2.FullPath(dir.Path), func() fs.Node {
+			return nil
+		})
+		dir.wfs.forgetNode(newPath)
+		dir.wfs.forgetNode(oldPath)
+		if oldFileNode != nil && newDirNode != nil {
+			oldFile := oldFileNode.(*File)
+			oldFile.Name = req.NewName
+			oldFile.dir = newDirNode.(*Dir)
+			dir.wfs.getNode(newPath, func() fs.Node {
+				return oldFile
+			})
+
+		}
+	}
+
+	return err
 }

weed/filesys/dirty_page.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -15,113 +15,68 @@ import (
 )
 
 type ContinuousDirtyPages struct {
-	hasData bool
-	Offset  int64
-	Size    int64
-	Data    []byte
-	f       *File
-	lock    sync.Mutex
+	intervals   *ContinuousIntervals
+	f           *File
+	lock        sync.Mutex
+	collection  string
+	replication string
 }
 
 func newDirtyPages(file *File) *ContinuousDirtyPages {
 	return &ContinuousDirtyPages{
-		Data: nil,
-		f:    file,
+		intervals: &ContinuousIntervals{},
+		f:         file,
 	}
 }
 
 func (pages *ContinuousDirtyPages) releaseResource() {
-	if pages.Data != nil {
-		pages.f.wfs.bufPool.Put(pages.Data)
-		pages.Data = nil
-		atomic.AddInt32(&counter, -1)
-		glog.V(3).Infof("%s/%s releasing resource %d", pages.f.dir.Path, pages.f.Name, counter)
-	}
 }
 
 var counter = int32(0)
 
-func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, data []byte) (chunks []*filer_pb.FileChunk, err error) {
+func (pages *ContinuousDirtyPages) AddPage(offset int64, data []byte) (chunks []*filer_pb.FileChunk, err error) {
 
 	pages.lock.Lock()
 	defer pages.lock.Unlock()
 
-	var chunk *filer_pb.FileChunk
+	glog.V(3).Infof("%s AddPage [%d,%d)", pages.f.fullpath(), offset, offset+int64(len(data)))
 
 	if len(data) > int(pages.f.wfs.option.ChunkSizeLimit) {
 		// this is more than what buffer can hold.
-		return pages.flushAndSave(ctx, offset, data)
+		return pages.flushAndSave(offset, data)
 	}
 
-	if pages.Data == nil {
-		pages.Data = pages.f.wfs.bufPool.Get().([]byte)
-		atomic.AddInt32(&counter, 1)
-		glog.V(3).Infof("%s/%s acquire resource %d", pages.f.dir.Path, pages.f.Name, counter)
-	}
+	pages.intervals.AddInterval(data, offset)
 
-	if offset < pages.Offset || offset >= pages.Offset+int64(len(pages.Data)) ||
-		pages.Offset+int64(len(pages.Data)) < offset+int64(len(data)) {
-		// if the data is out of range,
-		// or buffer is full if adding new data,
-		// flush current buffer and add new data
+	var chunk *filer_pb.FileChunk
+	var hasSavedData bool
 
-		// println("offset", offset, "size", len(data), "existing offset", pages.Offset, "size", pages.Size)
-
-		if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
-			if chunk != nil {
-				glog.V(4).Infof("%s/%s add save [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
-				chunks = append(chunks, chunk)
-			}
-		} else {
-			glog.V(0).Infof("%s/%s add save [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
-			return
+	if pages.intervals.TotalSize() > pages.f.wfs.option.ChunkSizeLimit {
+		chunk, hasSavedData, err = pages.saveExistingLargestPageToStorage()
+		if hasSavedData {
+			chunks = append(chunks, chunk)
 		}
-		pages.Offset = offset
-		copy(pages.Data, data)
-		pages.Size = int64(len(data))
-		return
 	}
 
-	if offset != pages.Offset+pages.Size {
-		// when this happens, debug shows the data overlapping with existing data is empty
-		// the data is not just append
-		if offset == pages.Offset && int(pages.Size) < len(data) {
-			// glog.V(2).Infof("pages[%d,%d) pages.Data len=%v, data len=%d, pages.Size=%d", pages.Offset, pages.Offset+pages.Size, len(pages.Data), len(data), pages.Size)
-			copy(pages.Data[pages.Size:], data[pages.Size:])
-		} else {
-			if pages.Size != 0 {
-				glog.V(1).Infof("%s/%s add page: pages [%d, %d) write [%d, %d)", pages.f.dir.Path, pages.f.Name, pages.Offset, pages.Offset+pages.Size, offset, offset+int64(len(data)))
-			}
-			return pages.flushAndSave(ctx, offset, data)
-		}
-	} else {
-		copy(pages.Data[offset-pages.Offset:], data)
-	}
-
-	pages.Size = max(pages.Size, offset+int64(len(data))-pages.Offset)
-
 	return
 }
 
-func (pages *ContinuousDirtyPages) flushAndSave(ctx context.Context, offset int64, data []byte) (chunks []*filer_pb.FileChunk, err error) {
+func (pages *ContinuousDirtyPages) flushAndSave(offset int64, data []byte) (chunks []*filer_pb.FileChunk, err error) {
 
 	var chunk *filer_pb.FileChunk
+	var newChunks []*filer_pb.FileChunk
 
 	// flush existing
-	if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
-		if chunk != nil {
-			glog.V(4).Infof("%s/%s flush existing [%d,%d) to %s", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.FileId)
-			chunks = append(chunks, chunk)
+	if newChunks, err = pages.saveExistingPagesToStorage(); err == nil {
+		if newChunks != nil {
+			chunks = append(chunks, newChunks...)
 		}
 	} else {
-		glog.V(0).Infof("%s/%s failed to flush1 [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
 		return
 	}
-	pages.Size = 0
-	pages.Offset = 0
 
 	// flush the new page
-	if chunk, err = pages.saveToStorage(ctx, data, offset); err == nil {
+	if chunk, err = pages.saveToStorage(bytes.NewReader(data), offset, int64(len(data))); err == nil {
 		if chunk != nil {
 			glog.V(4).Infof("%s/%s flush big request [%d,%d) to %s", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.FileId)
 			chunks = append(chunks, chunk)
@@ -134,40 +89,62 @@ func (pages *ContinuousDirtyPages) flushAndSave(ctx context.Context, offset int6
 	return
 }
 
-func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *filer_pb.FileChunk, err error) {
+func (pages *ContinuousDirtyPages) FlushToStorage() (chunks []*filer_pb.FileChunk, err error) {
 
 	pages.lock.Lock()
 	defer pages.lock.Unlock()
 
-	if pages.Size == 0 {
-		return nil, nil
-	}
+	return pages.saveExistingPagesToStorage()
+}
 
-	if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
-		pages.Size = 0
-		pages.Offset = 0
-		if chunk != nil {
-			glog.V(4).Infof("%s/%s flush [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
+func (pages *ContinuousDirtyPages) saveExistingPagesToStorage() (chunks []*filer_pb.FileChunk, err error) {
+
+	var hasSavedData bool
+	var chunk *filer_pb.FileChunk
+
+	for {
+
+		chunk, hasSavedData, err = pages.saveExistingLargestPageToStorage()
+		if !hasSavedData {
+			return chunks, err
+		}
+
+		if err == nil {
+			chunks = append(chunks, chunk)
+		} else {
+			return
 		}
 	}
+
+}
+
+func (pages *ContinuousDirtyPages) saveExistingLargestPageToStorage() (chunk *filer_pb.FileChunk, hasSavedData bool, err error) {
+
+	maxList := pages.intervals.RemoveLargestIntervalLinkedList()
+	if maxList == nil {
+		return nil, false, nil
+	}
+
+	chunk, err = pages.saveToStorage(maxList.ToReader(), maxList.Offset(), maxList.Size())
+	if err == nil {
+		hasSavedData = true
+		glog.V(3).Infof("%s saveToStorage [%d,%d) %s", pages.f.fullpath(), maxList.Offset(), maxList.Offset()+maxList.Size(), chunk.FileId)
+	} else {
+		glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), maxList.Offset(), maxList.Offset()+maxList.Size(), err)
+		return
+	}
+
 	return
 }
 
-func (pages *ContinuousDirtyPages) saveExistingPagesToStorage(ctx context.Context) (*filer_pb.FileChunk, error) {
-
-	if pages.Size == 0 {
-		return nil, nil
-	}
-
-	return pages.saveToStorage(ctx, pages.Data[:pages.Size], pages.Offset)
-}
-
-func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte, offset int64) (*filer_pb.FileChunk, error) {
+func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, size int64) (*filer_pb.FileChunk, error) {
 
 	var fileId, host string
 	var auth security.EncodedJwt
 
-	if err := pages.f.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	dir, _ := pages.f.fullpath().DirAndName()
+
+	if err := pages.f.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.AssignVolumeRequest{
 			Count:       1,
@@ -175,15 +152,21 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte
 			Collection:  pages.f.wfs.option.Collection,
 			TtlSec:      pages.f.wfs.option.TtlSec,
 			DataCenter:  pages.f.wfs.option.DataCenter,
+			ParentPath:  dir,
 		}
 
-		resp, err := client.AssignVolume(ctx, request)
+		resp, err := client.AssignVolume(context.Background(), request)
 		if err != nil {
 			glog.V(0).Infof("assign volume failure %v: %v", request, err)
 			return err
 		}
+		if resp.Error != "" {
+			return fmt.Errorf("assign volume failure %v: %v", request, resp.Error)
+		}
 
 		fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth)
+		host = pages.f.wfs.AdjustedUrl(host)
+		pages.collection, pages.replication = resp.Collection, resp.Replication
 
 		return nil
 	}); err != nil {
@@ -191,8 +174,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte
 	}
 
 	fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
-	bufReader := bytes.NewReader(buf)
-	uploadResult, err := operation.Upload(fileUrl, pages.f.Name, bufReader, false, "", nil, auth)
+	uploadResult, err := operation.Upload(fileUrl, pages.f.Name, pages.f.wfs.option.Cipher, reader, false, "", nil, auth)
 	if err != nil {
 		glog.V(0).Infof("upload data %v to %s: %v", pages.f.Name, fileUrl, err)
 		return nil, fmt.Errorf("upload data: %v", err)
@@ -203,11 +185,13 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte
 	}
 
 	return &filer_pb.FileChunk{
-		FileId: fileId,
-		Offset: offset,
-		Size:   uint64(len(buf)),
-		Mtime:  time.Now().UnixNano(),
-		ETag:   uploadResult.ETag,
+		FileId:    fileId,
+		Offset:    offset,
+		Size:      uint64(size),
+		Mtime:     time.Now().UnixNano(),
+		ETag:      uploadResult.ETag,
+		CipherKey: uploadResult.CipherKey,
+		IsGzipped: uploadResult.Gzip > 0,
 	}, nil
 
 }
@@ -218,3 +202,18 @@ func max(x, y int64) int64 {
 	}
 	return y
 }
+func min(x, y int64) int64 {
+	if x < y {
+		return x
+	}
+	return y
+}
+
+func (pages *ContinuousDirtyPages) ReadDirtyData(data []byte, startOffset int64) (offset int64, size int) {
+
+	pages.lock.Lock()
+	defer pages.lock.Unlock()
+
+	return pages.intervals.ReadData(data, startOffset)
+
+}

weed/filesys/dirty_page_interval.go

@@ -0,0 +1,220 @@
+package filesys
+
+import (
+	"bytes"
+	"io"
+	"math"
+)
+
+type IntervalNode struct {
+	Data   []byte
+	Offset int64
+	Size   int64
+	Next   *IntervalNode
+}
+
+type IntervalLinkedList struct {
+	Head *IntervalNode
+	Tail *IntervalNode
+}
+
+type ContinuousIntervals struct {
+	lists []*IntervalLinkedList
+}
+
+func (list *IntervalLinkedList) Offset() int64 {
+	return list.Head.Offset
+}
+func (list *IntervalLinkedList) Size() int64 {
+	return list.Tail.Offset + list.Tail.Size - list.Head.Offset
+}
+func (list *IntervalLinkedList) addNodeToTail(node *IntervalNode) {
+	// glog.V(4).Infof("add to tail [%d,%d) + [%d,%d) => [%d,%d)", list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, node.Offset+node.Size, list.Head.Offset, node.Offset+node.Size)
+	list.Tail.Next = node
+	list.Tail = node
+}
+func (list *IntervalLinkedList) addNodeToHead(node *IntervalNode) {
+	// glog.V(4).Infof("add to head [%d,%d) + [%d,%d) => [%d,%d)", node.Offset, node.Offset+node.Size, list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, list.Tail.Offset+list.Tail.Size)
+	node.Next = list.Head
+	list.Head = node
+}
+
+func (list *IntervalLinkedList) ReadData(buf []byte, start, stop int64) {
+	t := list.Head
+	for {
+
+		nodeStart, nodeStop := max(start, t.Offset), min(stop, t.Offset+t.Size)
+		if nodeStart < nodeStop {
+			// glog.V(0).Infof("copying start=%d stop=%d t=[%d,%d) t.data=%d => bufSize=%d nodeStart=%d, nodeStop=%d", start, stop, t.Offset, t.Offset+t.Size, len(t.Data), len(buf), nodeStart, nodeStop)
+			copy(buf[nodeStart-start:], t.Data[nodeStart-t.Offset:nodeStop-t.Offset])
+		}
+
+		if t.Next == nil {
+			break
+		}
+		t = t.Next
+	}
+}
+
+func (c *ContinuousIntervals) TotalSize() (total int64) {
+	for _, list := range c.lists {
+		total += list.Size()
+	}
+	return
+}
+
+func subList(list *IntervalLinkedList, start, stop int64) *IntervalLinkedList {
+	var nodes []*IntervalNode
+	for t := list.Head; t != nil; t = t.Next {
+		nodeStart, nodeStop := max(start, t.Offset), min(stop, t.Offset+t.Size)
+		if nodeStart >= nodeStop {
+			// skip non overlapping IntervalNode
+			continue
+		}
+		nodes = append(nodes, &IntervalNode{
+			Data:   t.Data[nodeStart-t.Offset : nodeStop-t.Offset],
+			Offset: nodeStart,
+			Size:   nodeStop - nodeStart,
+			Next:   nil,
+		})
+	}
+	for i := 1; i < len(nodes); i++ {
+		nodes[i-1].Next = nodes[i]
+	}
+	return &IntervalLinkedList{
+		Head: nodes[0],
+		Tail: nodes[len(nodes)-1],
+	}
+}
+
+func (c *ContinuousIntervals) AddInterval(data []byte, offset int64) {
+
+	interval := &IntervalNode{Data: data, Offset: offset, Size: int64(len(data))}
+
+	var newLists []*IntervalLinkedList
+	for _, list := range c.lists {
+		// if list is to the left of new interval, add to the new list
+		if list.Tail.Offset+list.Tail.Size <= interval.Offset {
+			newLists = append(newLists, list)
+		}
+		// if list is to the right of new interval, add to the new list
+		if interval.Offset+interval.Size <= list.Head.Offset {
+			newLists = append(newLists, list)
+		}
+		// if new interval overwrite the right part of the list
+		if list.Head.Offset < interval.Offset && interval.Offset < list.Tail.Offset+list.Tail.Size {
+			// create a new list of the left part of existing list
+			newLists = append(newLists, subList(list, list.Offset(), interval.Offset))
+		}
+		// if new interval overwrite the left part of the list
+		if list.Head.Offset < interval.Offset+interval.Size && interval.Offset+interval.Size < list.Tail.Offset+list.Tail.Size {
+			// create a new list of the right part of existing list
+			newLists = append(newLists, subList(list, interval.Offset+interval.Size, list.Tail.Offset+list.Tail.Size))
+		}
+		// skip anything that is fully overwritten by the new interval
+	}
+
+	c.lists = newLists
+	// add the new interval to the lists, connecting neighbor lists
+	var prevList, nextList *IntervalLinkedList
+
+	for _, list := range c.lists {
+		if list.Head.Offset == interval.Offset+interval.Size {
+			nextList = list
+			break
+		}
+	}
+
+	for _, list := range c.lists {
+		if list.Head.Offset+list.Size() == offset {
+			list.addNodeToTail(interval)
+			prevList = list
+			break
+		}
+	}
+
+	if prevList != nil && nextList != nil {
+		// glog.V(4).Infof("connecting [%d,%d) + [%d,%d) => [%d,%d)", prevList.Head.Offset, prevList.Tail.Offset+prevList.Tail.Size, nextList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size, prevList.Head.Offset, nextList.Tail.Offset+nextList.Tail.Size)
+		prevList.Tail.Next = nextList.Head
+		prevList.Tail = nextList.Tail
+		c.removeList(nextList)
+	} else if nextList != nil {
+		// add to head was not done when checking
+		nextList.addNodeToHead(interval)
+	}
+	if prevList == nil && nextList == nil {
+		c.lists = append(c.lists, &IntervalLinkedList{
+			Head: interval,
+			Tail: interval,
+		})
+	}
+
+	return
+}
+
+func (c *ContinuousIntervals) RemoveLargestIntervalLinkedList() *IntervalLinkedList {
+	var maxSize int64
+	maxIndex := -1
+	for k, list := range c.lists {
+		if maxSize <= list.Size() {
+			maxSize = list.Size()
+			maxIndex = k
+		}
+	}
+	if maxSize <= 0 {
+		return nil
+	}
+
+	t := c.lists[maxIndex]
+	c.lists = append(c.lists[0:maxIndex], c.lists[maxIndex+1:]...)
+	return t
+
+}
+
+func (c *ContinuousIntervals) removeList(target *IntervalLinkedList) {
+	index := -1
+	for k, list := range c.lists {
+		if list.Offset() == target.Offset() {
+			index = k
+		}
+	}
+	if index < 0 {
+		return
+	}
+
+	c.lists = append(c.lists[0:index], c.lists[index+1:]...)
+
+}
+
+func (c *ContinuousIntervals) ReadData(data []byte, startOffset int64) (offset int64, size int) {
+	var minOffset int64 = math.MaxInt64
+	var maxStop int64
+	for _, list := range c.lists {
+		start := max(startOffset, list.Offset())
+		stop := min(startOffset+int64(len(data)), list.Offset()+list.Size())
+		if start <= stop {
+			list.ReadData(data[start-startOffset:], start, stop)
+			minOffset = min(minOffset, start)
+			maxStop = max(maxStop, stop)
+		}
+	}
+
+	if minOffset == math.MaxInt64 {
+		return 0, 0
+	}
+
+	offset = minOffset
+	size = int(maxStop - offset)
+	return
+}
+
+func (l *IntervalLinkedList) ToReader() io.Reader {
+	var readers []io.Reader
+	t := l.Head
+	readers = append(readers, bytes.NewReader(t.Data))
+	for t.Next != nil {
+		t = t.Next
+		readers = append(readers, bytes.NewReader(t.Data))
+	}
+	return io.MultiReader(readers...)
+}

weed/filesys/dirty_page_interval_test.go

@@ -0,0 +1,72 @@
+package filesys
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestContinuousIntervals_AddIntervalAppend(t *testing.T) {
+
+	c := &ContinuousIntervals{}
+
+	// 25, 25, 25
+	c.AddInterval(getBytes(25, 3), 0)
+	//  _,  _, 23, 23, 23, 23
+	c.AddInterval(getBytes(23, 4), 2)
+
+	expectedData(t, c, 0, 25, 25, 23, 23, 23, 23)
+
+}
+
+func TestContinuousIntervals_AddIntervalInnerOverwrite(t *testing.T) {
+
+	c := &ContinuousIntervals{}
+
+	// 25, 25, 25, 25, 25
+	c.AddInterval(getBytes(25, 5), 0)
+	//  _,  _, 23, 23
+	c.AddInterval(getBytes(23, 2), 2)
+
+	expectedData(t, c, 0, 25, 25, 23, 23, 25)
+
+}
+
+func TestContinuousIntervals_AddIntervalFullOverwrite(t *testing.T) {
+
+	c := &ContinuousIntervals{}
+
+	// 25,
+	c.AddInterval(getBytes(25, 1), 0)
+	//  _,  _,  _,  _, 23, 23
+	c.AddInterval(getBytes(23, 2), 4)
+	//  _,  _,  _, 24, 24, 24, 24
+	c.AddInterval(getBytes(24, 4), 3)
+
+	//  _,  22, 22
+	c.AddInterval(getBytes(22, 2), 1)
+
+	expectedData(t, c, 0, 25, 22, 22, 24, 24, 24, 24)
+
+}
+
+func expectedData(t *testing.T, c *ContinuousIntervals, offset int, data ...byte) {
+	start, stop := int64(offset), int64(offset+len(data))
+	for _, list := range c.lists {
+		nodeStart, nodeStop := max(start, list.Head.Offset), min(stop, list.Head.Offset+list.Size())
+		if nodeStart < nodeStop {
+			buf := make([]byte, nodeStop-nodeStart)
+			list.ReadData(buf, nodeStart, nodeStop)
+			if bytes.Compare(buf, data[nodeStart-start:nodeStop-start]) != 0 {
+				t.Errorf("expected %v actual %v", data[nodeStart-start:nodeStop-start], buf)
+			}
+		}
+	}
+}
+
+func getBytes(content byte, length int) []byte {
+	data := make([]byte, length)
+	for i := 0; i < length; i++ {
+		data[i] = content
+	}
+	return data
+}

weed/filesys/file.go

@@ -3,7 +3,6 @@ package filesys
 import (
 	"context"
 	"os"
-	"path/filepath"
 	"sort"
 	"time"
 
@@ -24,6 +23,7 @@ var _ = fs.NodeGetxattrer(&File{})
 var _ = fs.NodeSetxattrer(&File{})
 var _ = fs.NodeRemovexattrer(&File{})
 var _ = fs.NodeListxattrer(&File{})
+var _ = fs.NodeForgetter(&File{})
 
 type File struct {
 	Name           string
@@ -31,23 +31,32 @@ type File struct {
 	wfs            *WFS
 	entry          *filer_pb.Entry
 	entryViewCache []filer2.VisibleInterval
-	isOpen         bool
+	isOpen         int
 }
 
-func (file *File) fullpath() string {
-	return filepath.Join(file.dir.Path, file.Name)
+func (file *File) fullpath() filer2.FullPath {
+	return filer2.NewFullPath(file.dir.Path, file.Name)
 }
 
 func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error {
 
-	glog.V(4).Infof("file Attr %s", file.fullpath())
+	glog.V(4).Infof("file Attr %s, open:%v, existing attr: %+v", file.fullpath(), file.isOpen, attr)
 
-	if err := file.maybeLoadEntry(ctx); err != nil {
-		return err
+	if file.isOpen <= 0 {
+		if err := file.maybeLoadEntry(ctx); err != nil {
+			return err
+		}
 	}
 
+	attr.Inode = file.fullpath().AsInode()
+	attr.Valid = time.Second
 	attr.Mode = os.FileMode(file.entry.Attributes.FileMode)
 	attr.Size = filer2.TotalSize(file.entry.Chunks)
+	if file.isOpen > 0 {
+		attr.Size = file.entry.Attributes.FileSize
+		glog.V(4).Infof("file Attr %s, open:%v, size: %d", file.fullpath(), file.isOpen, attr.Size)
+	}
+	attr.Crtime = time.Unix(file.entry.Attributes.Crtime, 0)
 	attr.Mtime = time.Unix(file.entry.Attributes.Mtime, 0)
 	attr.Gid = file.entry.Attributes.Gid
 	attr.Uid = file.entry.Attributes.Uid
@@ -60,7 +69,7 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error {
 
 func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
 
-	glog.V(4).Infof("file Getxattr %s", file.fullpath())
+	// glog.V(4).Infof("file Getxattr %s", file.fullpath())
 
 	if err := file.maybeLoadEntry(ctx); err != nil {
 		return err
@@ -73,7 +82,7 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op
 
 	glog.V(4).Infof("file %v open %+v", file.fullpath(), req)
 
-	file.isOpen = true
+	file.isOpen++
 
 	handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid)
 
@@ -87,17 +96,28 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op
 
 func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
 
+	glog.V(3).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.entry.Attributes)
+
 	if err := file.maybeLoadEntry(ctx); err != nil {
 		return err
 	}
 
-	glog.V(3).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.entry.Attributes)
 	if req.Valid.Size() {
 
 		glog.V(3).Infof("%v file setattr set size=%v", file.fullpath(), req.Size)
-		if req.Size == 0 {
+		if req.Size < filer2.TotalSize(file.entry.Chunks) {
 			// fmt.Printf("truncate %v \n", fullPath)
-			file.entry.Chunks = nil
+			var chunks []*filer_pb.FileChunk
+			for _, chunk := range file.entry.Chunks {
+				int64Size := int64(chunk.Size)
+				if chunk.Offset+int64Size > int64(req.Size) {
+					int64Size = int64(req.Size) - chunk.Offset
+				}
+				if int64Size > 0 {
+					chunks = append(chunks, chunk)
+				}
+			}
+			file.entry.Chunks = chunks
 			file.entryViewCache = nil
 		}
 		file.entry.Attributes.FileSize = req.Size
@@ -122,13 +142,13 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		file.entry.Attributes.Mtime = req.Mtime.Unix()
 	}
 
-	if file.isOpen {
+	if file.isOpen > 0 {
 		return nil
 	}
 
-	file.wfs.listDirectoryEntriesCache.Delete(file.fullpath())
+	file.wfs.cacheDelete(file.fullpath())
 
-	return file.saveEntry(ctx)
+	return file.saveEntry()
 
 }
 
@@ -144,7 +164,9 @@ func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error
 		return err
 	}
 
-	return file.saveEntry(ctx)
+	file.wfs.cacheDelete(file.fullpath())
+
+	return file.saveEntry()
 
 }
 
@@ -160,7 +182,9 @@ func (file *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest)
 		return err
 	}
 
-	return file.saveEntry(ctx)
+	file.wfs.cacheDelete(file.fullpath())
+
+	return file.saveEntry()
 
 }
 
@@ -176,7 +200,7 @@ func (file *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, res
 		return err
 	}
 
-	return file.saveEntry(ctx)
+	return nil
 
 }
 
@@ -188,9 +212,16 @@ func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
 	return nil
 }
 
+func (file *File) Forget() {
+	glog.V(3).Infof("Forget file %s/%s", file.dir.Path, file.Name)
+
+	file.wfs.forgetNode(filer2.NewFullPath(file.dir.Path, file.Name))
+
+}
+
 func (file *File) maybeLoadEntry(ctx context.Context) error {
-	if file.entry == nil || !file.isOpen {
-		entry, err := file.wfs.maybeLoadEntry(ctx, file.dir.Path, file.Name)
+	if file.entry == nil || file.isOpen <= 0 {
+		entry, err := file.wfs.maybeLoadEntry(file.dir.Path, file.Name)
 		if err != nil {
 			return err
 		}
@@ -201,12 +232,6 @@ func (file *File) maybeLoadEntry(ctx context.Context) error {
 	return nil
 }
 
-func (file *File) addChunk(chunk *filer_pb.FileChunk) {
-	if chunk != nil {
-		file.addChunks([]*filer_pb.FileChunk{chunk})
-	}
-}
-
 func (file *File) addChunks(chunks []*filer_pb.FileChunk) {
 
 	sort.Slice(chunks, func(i, j int) bool {
@@ -231,8 +256,8 @@ func (file *File) setEntry(entry *filer_pb.Entry) {
 	file.entryViewCache = filer2.NonOverlappingVisibleIntervals(file.entry.Chunks)
 }
 
-func (file *File) saveEntry(ctx context.Context) error {
-	return file.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+func (file *File) saveEntry() error {
+	return file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.UpdateEntryRequest{
 			Directory: file.dir.Path,
@@ -240,7 +265,7 @@ func (file *File) saveEntry(ctx context.Context) error {
 		}
 
 		glog.V(1).Infof("save file entry: %v", request)
-		_, err := client.UpdateEntry(ctx, request)
+		_, err := client.UpdateEntry(context.Background(), request)
 		if err != nil {
 			glog.V(0).Infof("UpdateEntry file %s/%s: %v", file.dir.Path, file.Name, err)
 			return fuse.EIO