2.80

add s3.clean.uploads -timeAgo=24h
skip the rest logic
2026-10-02 04:32:42 +00:00 · 2021-11-29 00:57:08 -08:00 · 2021-11-29 00:49:49 -08:00 · 2021-11-29 00:32:21 -08:00 · 2021-11-29 00:28:26 -08:00 · 2021-11-28 23:50:52 -08:00
125 changed files with 3666 additions and 2392 deletions
@@ -37,14 +37,14 @@ jobs:
        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}

      - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          release_tag: dev
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
@@ -53,14 +53,14 @@ jobs:
          asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"

      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          release_tag: dev
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
@@ -84,14 +84,14 @@ jobs:
        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}

      - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          release_tag: dev
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
@@ -100,14 +100,14 @@ jobs:
          asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"

      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          release_tag: dev
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
          project_path: weed
@@ -25,13 +25,13 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          # build_flags: -tags 5BytesOffset # optional, default is
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
@@ -39,13 +39,13 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
@@ -25,13 +25,13 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          # build_flags: -tags 5BytesOffset # optional, default is
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
@@ -39,13 +39,13 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
@@ -25,13 +25,13 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          # build_flags: -tags 5BytesOffset # optional, default is
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
@@ -39,13 +39,13 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
@@ -25,13 +25,13 @@ jobs:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          # build_flags: -tags 5BytesOffset # optional, default is
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
@@ -39,13 +39,13 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
@@ -2,7 +2,8 @@ name: "docker: build latest container"

 on:
  push:
-    branches: [ master ]
+    tags:
+      - '*'
  workflow_dispatch: []

 jobs:
@@ -23,7 +24,7 @@ jobs:
            chrislusf/seaweedfs
            ghcr.io/chrislusf/seaweedfs
          tags: |
-            type=raw,value=dev
+            type=raw,value=latest
          labels: |
            org.opencontainers.image.title=seaweedfs
            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
@@ -5,12 +5,16 @@

 <h2 align="center">Generous Backers ($50+)</h2>

- [4Sight Imaging](https://www.4sightimaging.com/)
 - [Evercam Camera Management Software](https://evercam.io/)
- [Admiral](https://getadmiral.com)

 <h2 align="center">Backers</h2>

 - [ColorfulClouds Tech Co. Ltd.](https://caiyunai.com/)
 - [Haravan - Ecommerce Platform](https://www.haravan.com)
 - PeterCxy - Creator of Shelter App
+- [Hive Games](https://playhive.com/)
+- Flowm
+- Yoni Nakache
+- Catalin Constantin
+- MingLi Yuan
+- Leroy van Logchem
@@ -48,6 +48,7 @@ EXPOSE 7333
 RUN mkdir -p /data/filerldb2

 VOLUME /data
+WORKDIR /data

 COPY filer.toml /etc/seaweedfs/filer.toml
 COPY entrypoint.sh /entrypoint.sh
@@ -37,6 +37,7 @@ EXPOSE 7333
 RUN mkdir -p /data/filerldb2

 VOLUME /data
+WORKDIR /data

 RUN chmod +x /entrypoint.sh

@@ -1,4 +1,4 @@
-FROM amd64/golang:1.17-alpine as builder
+FROM golang:1.17-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
@@ -36,6 +36,7 @@ EXPOSE 7333
 RUN mkdir -p /data/filerldb2

 VOLUME /data
+WORKDIR /data

 RUN chmod +x /entrypoint.sh

@@ -1,4 +1,4 @@
-FROM amd64/golang:1.17-alpine as builder
+FROM golang:1.17-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
@@ -36,6 +36,7 @@ EXPOSE 7333
 RUN mkdir -p /data/filerldb2

 VOLUME /data
+WORKDIR /data

 RUN chmod +x /entrypoint.sh

@@ -26,6 +26,7 @@ EXPOSE 7333
 RUN mkdir -p /data/filerldb2

 VOLUME /data
+WORKDIR /data

 RUN chmod +x /entrypoint.sh

@@ -54,6 +54,8 @@ RUN mkdir -p /data/filerldb2

 VOLUME /data

+WORKDIR /data
+
 RUN chmod +x /entrypoint.sh

 ENTRYPOINT ["/entrypoint.sh"]
@@ -112,9 +112,9 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
-	github.com/tidwall/gjson v1.8.1
-	github.com/tidwall/match v1.0.3
-	github.com/tidwall/pretty v1.1.0 // indirect
+	github.com/tidwall/gjson v1.10.2
+	github.com/tidwall/match v1.1.1
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
 	github.com/valyala/bytebufferpool v1.0.0
@@ -778,11 +778,17 @@ github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7
 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
 github.com/tidwall/gjson v1.8.1 h1:8j5EE9Hrh3l9Od1OIEDAb7IpezNA20UdRngNAj5N0WU=
 github.com/tidwall/gjson v1.8.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
+github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo=
+github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
 github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
 github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 h1:6iRwZdrFUzbcVYZwa8dXTIILGIxmmhjyUPJEcwzPGaU=
@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.72"
-version: "2.72"
+appVersion: "2.80"
+version: "2.80"
@@ -133,11 +133,6 @@ spec:
              -encryptVolumeData \
              {{- end }}
              -ip=${POD_IP} \
-              {{- if .Values.filer.enable_peers }}
-              {{- if gt (.Values.filer.replicas | int) 1 }}
-              -peers=$(echo -n "{{ range $index := until (.Values.filer.replicas | int) }}${SEAWEEDFS_FULLNAME}-filer-{{ $index }}.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}{{ if lt $index (sub ($.Values.filer.replicas | int) 1) }},{{ end }}{{ end }}" | sed "s/$HOSTNAME.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}//" | sed 's/,$//; 's/^,//'; s/,,/,/;' )  \
-              {{- end }}
-              {{- end }}
              {{- if .Values.filer.s3.enabled }}
              -s3 \
              -s3.port={{ .Values.filer.s3.port }} \
@@ -1,61 +1,67 @@
+{{- if .Values.filer.ingress.enabled }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1beta1
+{{- else }}
 apiVersion: extensions/v1beta1
+{{- end }}
 kind: Ingress
 metadata:
- name: ingress-{{ template "seaweedfs.name" . }}-filer
- namespace: {{ .Release.Namespace }}
- annotations:
-   kubernetes.io/ingress.class: "nginx"
-   nginx.ingress.kubernetes.io/auth-type: "basic"
-   nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
-   nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Filer'
-   nginx.ingress.kubernetes.io/service-upstream: "true"
-   nginx.ingress.kubernetes.io/rewrite-target: /$1
-   nginx.ingress.kubernetes.io/use-regex: "true"
-   nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
-   nginx.ingress.kubernetes.io/ssl-redirect: "false"
-   nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
-   nginx.ingress.kubernetes.io/configuration-snippet: |
-     sub_filter '<head>' '<head> <base href="/sw-filer/">'; #add base url
-     sub_filter '="/' '="./';                               #make absolute paths to relative
-     sub_filter '=/' '=./';
-     sub_filter '/seaweedfsstatic' './seaweedfsstatic';
-     sub_filter_once off;
+  name: ingress-{{ template "seaweedfs.name" . }}-filer
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    {{ omit .Values.filer.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }}
 spec:
- rules:
- - http:
-     paths:
-     - path: /sw-filer/?(.*)
-       backend:
-         serviceName: {{ template "seaweedfs.name" . }}-filer
-         servicePort: {{ .Values.filer.port }}
+  ingressClassName: {{ .Values.filer.ingress.className | quote }}
+  rules:
+  - http:
+      paths:
+      - path: /sw-filer/?(.*)
+        pathType: ImplementationSpecific
+        backend:
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+          service:
+            name: {{ template "seaweedfs.name" . }}-filer
+            port:
+              number: {{ .Values.filer.port }}
+              #name:
+{{- else }}
+          serviceName: {{ template "seaweedfs.name" . }}-filer
+          servicePort: {{ .Values.filer.port }}
+{{- end }}
+{{- end }}
 ---
+{{- if .Values.master.ingress.enabled }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1beta1
+{{- else }}
 apiVersion: extensions/v1beta1
+{{- end }}
 kind: Ingress
 metadata:
  name: ingress-{{ template "seaweedfs.name" . }}-master
  namespace: {{ .Release.Namespace }}
  annotations:
-    kubernetes.io/ingress.class: "nginx"
-    nginx.ingress.kubernetes.io/auth-type: "basic"
-    nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
-    nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master'
-    nginx.ingress.kubernetes.io/service-upstream: "true"
-    nginx.ingress.kubernetes.io/rewrite-target: /$1
-    nginx.ingress.kubernetes.io/use-regex: "true"
-    nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
-    nginx.ingress.kubernetes.io/ssl-redirect: "false"
-    nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
-    nginx.ingress.kubernetes.io/configuration-snippet: |
-      sub_filter '<head>' '<head> <base href="/sw-master/">'; #add base url
-      sub_filter '="/' '="./';                                #make absolute paths to relative
-      sub_filter '=/' '=./';
-      sub_filter '/seaweedfsstatic' './seaweedfsstatic';
-      sub_filter_once off;
+    {{ omit .Values.master.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }}
 spec:
+  ingressClassName: {{ .Values.master.ingress.className | quote }}
  rules:
  - http:
      paths:
      - path: /sw-master/?(.*)
+        pathType: ImplementationSpecific
        backend:
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+          service:
+            name: {{ template "seaweedfs.name" . }}-master
+            port:
+              number: {{ .Values.master.port }}
+              #name:
+{{- else }}
          serviceName: {{ template "seaweedfs.name" . }}-master
          servicePort: {{ .Values.master.port }}
+{{- end }}
+{{- end }}
@@ -1,7 +1,7 @@
 #hack for delete pod master after migration
 ---
 kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: seaweefds-rw-cr
 rules:
@@ -16,7 +16,7 @@ metadata:
  namespace: {{ .Release.Namespace }}
 ---
 kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: system:serviceaccount:seaweefds-rw-sa:default
 subjects:
@@ -26,4 +26,4 @@ subjects:
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
-  name: seaweefds-rw-cr
+  name: seaweefds-rw-cr
@@ -109,6 +109,26 @@ master:
  # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
  priorityClassName: ""

+  ingress:
+    enabled: false
+    className: "nginx"
+    annotations:
+      nginx.ingress.kubernetes.io/auth-type: "basic"
+      nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
+      nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master'
+      nginx.ingress.kubernetes.io/service-upstream: "true"
+      nginx.ingress.kubernetes.io/rewrite-target: /$1
+      nginx.ingress.kubernetes.io/use-regex: "true"
+      nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
+      nginx.ingress.kubernetes.io/ssl-redirect: "false"
+      nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+      nginx.ingress.kubernetes.io/configuration-snippet: |
+        sub_filter '<head>' '<head> <base href="/sw-master/">'; #add base url
+        sub_filter '="/' '="./';                                #make absolute paths to relative
+        sub_filter '=/' '=./';
+        sub_filter '/seaweedfsstatic' './seaweedfsstatic';
+        sub_filter_once off;
+
  extraEnvironmentVars:
    WEED_MASTER_VOLUME_GROWTH_COPY_1: 7
    WEED_MASTER_VOLUME_GROWTH_COPY_2: 6
@@ -246,8 +266,6 @@ filer:
  maxMB: null
  # encrypt data on volume servers
  encryptVolumeData: false
-  # enable peers sync metadata, for leveldb (localdb for filer but with sync across)
-  enable_peers: false

  # Whether proxy or redirect to volume server during file GET request
  redirectOnRead: false
@@ -311,6 +329,26 @@ filer:
  # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
  priorityClassName: ""

+  ingress:
+    enabled: false
+    className: "nginx"
+    annotations:
+      nginx.ingress.kubernetes.io/auth-type: "basic"
+      nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
+      nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Filer'
+      nginx.ingress.kubernetes.io/service-upstream: "true"
+      nginx.ingress.kubernetes.io/rewrite-target: /$1
+      nginx.ingress.kubernetes.io/use-regex: "true"
+      nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
+      nginx.ingress.kubernetes.io/ssl-redirect: "false"
+      nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+      nginx.ingress.kubernetes.io/configuration-snippet: |
+        sub_filter '<head>' '<head> <base href="/sw-filer/">'; #add base url
+        sub_filter '="/' '="./';                               #make absolute paths to relative
+        sub_filter '=/' '=./';
+        sub_filter '/seaweedfsstatic' './seaweedfsstatic';
+        sub_filter_once off;
+
  # extraEnvVars is a list of extra enviroment variables to set with the stateful set.
  extraEnvironmentVars:
    WEED_MYSQL_ENABLED: "true"
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.8</version>
+    <version>1.7.0</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -60,7 +60,7 @@
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpmime</artifactId>
-            <version>4.5.6</version>
+            <version>4.5.13</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.8</version>
+    <version>1.7.0</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -60,7 +60,7 @@
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpmime</artifactId>
-            <version>4.5.6</version>
+            <version>4.5.13</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.8</version>
+    <version>1.7.0</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -60,7 +60,7 @@
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpmime</artifactId>
-            <version>4.5.6</version>
+            <version>4.5.13</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
@@ -14,10 +14,10 @@ public class RemoteUtil {
        String dir = SeaweedOutputStream.getParentDirectory(fullpath);
        String name = SeaweedOutputStream.getFileName(fullpath);

-        final FilerProto.DownloadToLocalResponse downloadToLocalResponse = filerClient.getBlockingStub()
-                .downloadToLocal(FilerProto.DownloadToLocalRequest.newBuilder()
+        final FilerProto.CacheRemoteObjectToLocalClusterResponse response = filerClient.getBlockingStub()
+                .cacheRemoteObjectToLocalCluster(FilerProto.CacheRemoteObjectToLocalClusterRequest.newBuilder()
                .setDirectory(dir).setName(name).build());

-        return downloadToLocalResponse.getEntry();
+        return response.getEntry();
    }
 }
@@ -69,7 +69,7 @@ service SeaweedFiler {
    rpc KvPut (KvPutRequest) returns (KvPutResponse) {
    }

-    rpc DownloadToLocal (DownloadToLocalRequest) returns (DownloadToLocalResponse) {
+    rpc CacheRemoteObjectToLocalCluster (CacheRemoteObjectToLocalClusterRequest) returns (CacheRemoteObjectToLocalClusterResponse) {
    }
 }

@@ -403,10 +403,10 @@ message FilerConf {
 /////////////////////////
 // Remote Storage related
 /////////////////////////
-message DownloadToLocalRequest {
+message CacheRemoteObjectToLocalClusterRequest {
    string directory = 1;
    string name = 2;
 }
-message DownloadToLocalResponse {
+message CacheRemoteObjectToLocalClusterResponse {
    Entry entry = 1;
 }
@@ -11,13 +11,13 @@
        <dependency>
            <groupId>com.github.chrislusf</groupId>
            <artifactId>seaweedfs-client</artifactId>
-            <version>1.6.8</version>
+            <version>1.7.0</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.github.chrislusf</groupId>
            <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>1.6.8</version>
+            <version>1.7.0</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
@@ -301,7 +301,7 @@
    </snapshotRepository>
  </distributionManagement>
  <properties>
-    <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
+    <seaweedfs.client.version>1.7.0</seaweedfs.client.version>
    <hadoop.version>2.9.2</hadoop.version>
  </properties>
 </project>
@@ -5,7 +5,7 @@
    <modelVersion>4.0.0</modelVersion>

    <properties>
-        <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
+        <seaweedfs.client.version>1.7.0</seaweedfs.client.version>
        <hadoop.version>2.9.2</hadoop.version>
    </properties>

@@ -309,7 +309,7 @@
    </snapshotRepository>
  </distributionManagement>
  <properties>
-    <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
+    <seaweedfs.client.version>1.7.0</seaweedfs.client.version>
    <hadoop.version>3.1.1</hadoop.version>
  </properties>
 </project>
@@ -5,7 +5,7 @@
    <modelVersion>4.0.0</modelVersion>

    <properties>
-        <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
+        <seaweedfs.client.version>1.7.0</seaweedfs.client.version>
        <hadoop.version>3.1.1</hadoop.version>
    </properties>

@@ -2,10 +2,13 @@ BINARY = weed

 SOURCE_DIR = .

-all: debug_mount
+all: install

 .PHONY : clean debug_mount

+install:
+	go install
+
 clean:
 	go clean $(SOURCE_DIR)
 	rm -f $(BINARY)
@@ -0,0 +1,264 @@
+package cluster
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"math"
+	"sync"
+	"time"
+)
+
+const (
+	MasterType = "master"
+	FilerType  = "filer"
+	BrokerType = "broker"
+)
+
+type ClusterNode struct {
+	Address   pb.ServerAddress
+	Version   string
+	counter   int
+	createdTs time.Time
+}
+
+type Leaders struct {
+	leaders [3]pb.ServerAddress
+}
+
+type Cluster struct {
+	filers       map[pb.ServerAddress]*ClusterNode
+	filersLock   sync.RWMutex
+	filerLeaders *Leaders
+	brokers      map[pb.ServerAddress]*ClusterNode
+	brokersLock  sync.RWMutex
+}
+
+func NewCluster() *Cluster {
+	return &Cluster{
+		filers:       make(map[pb.ServerAddress]*ClusterNode),
+		filerLeaders: &Leaders{},
+		brokers:      make(map[pb.ServerAddress]*ClusterNode),
+	}
+}
+
+func (cluster *Cluster) AddClusterNode(nodeType string, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
+	switch nodeType {
+	case FilerType:
+		cluster.filersLock.Lock()
+		defer cluster.filersLock.Unlock()
+		if existingNode, found := cluster.filers[address]; found {
+			existingNode.counter++
+			return nil
+		}
+		cluster.filers[address] = &ClusterNode{
+			Address:   address,
+			Version:   version,
+			counter:   1,
+			createdTs: time.Now(),
+		}
+		return cluster.ensureFilerLeaders(true, nodeType, address)
+	case BrokerType:
+		cluster.brokersLock.Lock()
+		defer cluster.brokersLock.Unlock()
+		if existingNode, found := cluster.brokers[address]; found {
+			existingNode.counter++
+			return nil
+		}
+		cluster.brokers[address] = &ClusterNode{
+			Address:   address,
+			Version:   version,
+			counter:   1,
+			createdTs: time.Now(),
+		}
+		return []*master_pb.KeepConnectedResponse{
+			{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsAdd:    true,
+				},
+			},
+		}
+	case MasterType:
+	}
+	return nil
+}
+
+func (cluster *Cluster) RemoveClusterNode(nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
+	switch nodeType {
+	case FilerType:
+		cluster.filersLock.Lock()
+		defer cluster.filersLock.Unlock()
+		if existingNode, found := cluster.filers[address]; !found {
+			return nil
+		} else {
+			existingNode.counter--
+			if existingNode.counter <= 0 {
+				delete(cluster.filers, address)
+				return cluster.ensureFilerLeaders(false, nodeType, address)
+			}
+		}
+	case BrokerType:
+		cluster.brokersLock.Lock()
+		defer cluster.brokersLock.Unlock()
+		if existingNode, found := cluster.brokers[address]; !found {
+			return nil
+		} else {
+			existingNode.counter--
+			if existingNode.counter <= 0 {
+				delete(cluster.brokers, address)
+				return []*master_pb.KeepConnectedResponse{
+					{
+						ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+							NodeType: nodeType,
+							Address:  string(address),
+							IsAdd:    false,
+						},
+					},
+				}
+			}
+		}
+	case MasterType:
+	}
+	return nil
+}
+
+func (cluster *Cluster) ListClusterNode(nodeType string) (nodes []*ClusterNode) {
+	switch nodeType {
+	case FilerType:
+		cluster.filersLock.RLock()
+		defer cluster.filersLock.RUnlock()
+		for _, node := range cluster.filers {
+			nodes = append(nodes, node)
+		}
+	case BrokerType:
+		cluster.brokersLock.RLock()
+		defer cluster.brokersLock.RUnlock()
+		for _, node := range cluster.brokers {
+			nodes = append(nodes, node)
+		}
+	case MasterType:
+	}
+	return
+}
+
+func (cluster *Cluster) IsOneLeader(address pb.ServerAddress) bool {
+	return cluster.filerLeaders.isOneLeader(address)
+}
+
+func (cluster *Cluster) ensureFilerLeaders(isAdd bool, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
+	if isAdd {
+		if cluster.filerLeaders.addLeaderIfVacant(address) {
+			// has added the address as one leader
+			result = append(result, &master_pb.KeepConnectedResponse{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsLeader: true,
+					IsAdd:    true,
+				},
+			})
+		} else {
+			result = append(result, &master_pb.KeepConnectedResponse{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsLeader: false,
+					IsAdd:    true,
+				},
+			})
+		}
+	} else {
+		if cluster.filerLeaders.removeLeaderIfExists(address) {
+
+			result = append(result, &master_pb.KeepConnectedResponse{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsLeader: true,
+					IsAdd:    false,
+				},
+			})
+
+			// pick the freshest one, since it is less likely to go away
+			var shortestDuration int64 = math.MaxInt64
+			now := time.Now()
+			var candidateAddress pb.ServerAddress
+			for _, node := range cluster.filers {
+				if cluster.filerLeaders.isOneLeader(node.Address) {
+					continue
+				}
+				duration := now.Sub(node.createdTs).Nanoseconds()
+				if duration < shortestDuration {
+					shortestDuration = duration
+					candidateAddress = node.Address
+				}
+			}
+			if candidateAddress != "" {
+				cluster.filerLeaders.addLeaderIfVacant(candidateAddress)
+				// added a new leader
+				result = append(result, &master_pb.KeepConnectedResponse{
+					ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+						NodeType: nodeType,
+						Address:  string(candidateAddress),
+						IsLeader: true,
+						IsAdd:    true,
+					},
+				})
+			}
+		} else {
+			result = append(result, &master_pb.KeepConnectedResponse{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsLeader: false,
+					IsAdd:    false,
+				},
+			})
+		}
+	}
+	return
+}
+
+func (leaders *Leaders) addLeaderIfVacant(address pb.ServerAddress) (hasChanged bool) {
+	if leaders.isOneLeader(address) {
+		return
+	}
+	for i := 0; i < len(leaders.leaders); i++ {
+		if leaders.leaders[i] == "" {
+			leaders.leaders[i] = address
+			hasChanged = true
+			return
+		}
+	}
+	return
+}
+func (leaders *Leaders) removeLeaderIfExists(address pb.ServerAddress) (hasChanged bool) {
+	if !leaders.isOneLeader(address) {
+		return
+	}
+	for i := 0; i < len(leaders.leaders); i++ {
+		if leaders.leaders[i] == address {
+			leaders.leaders[i] = ""
+			hasChanged = true
+			return
+		}
+	}
+	return
+}
+func (leaders *Leaders) isOneLeader(address pb.ServerAddress) bool {
+	for i := 0; i < len(leaders.leaders); i++ {
+		if leaders.leaders[i] == address {
+			return true
+		}
+	}
+	return false
+}
+func (leaders *Leaders) GetLeaders() (addresses []pb.ServerAddress) {
+	for i := 0; i < len(leaders.leaders); i++ {
+		if leaders.leaders[i] != "" {
+			addresses = append(addresses, leaders.leaders[i])
+		}
+	}
+	return
+}
@@ -0,0 +1,47 @@
+package cluster
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestClusterAddRemoveNodes(t *testing.T) {
+	c := NewCluster()
+
+	c.AddClusterNode("filer", pb.ServerAddress("111:1"), "23.45")
+	c.AddClusterNode("filer", pb.ServerAddress("111:2"), "23.45")
+	assert.Equal(t, []pb.ServerAddress{
+		pb.ServerAddress("111:1"),
+		pb.ServerAddress("111:2"),
+	}, c.filerLeaders.GetLeaders())
+
+	c.AddClusterNode("filer", pb.ServerAddress("111:3"), "23.45")
+	c.AddClusterNode("filer", pb.ServerAddress("111:4"), "23.45")
+	assert.Equal(t, []pb.ServerAddress{
+		pb.ServerAddress("111:1"),
+		pb.ServerAddress("111:2"),
+		pb.ServerAddress("111:3"),
+	}, c.filerLeaders.GetLeaders())
+
+	c.AddClusterNode("filer", pb.ServerAddress("111:5"), "23.45")
+	c.AddClusterNode("filer", pb.ServerAddress("111:6"), "23.45")
+	c.RemoveClusterNode("filer", pb.ServerAddress("111:4"))
+	assert.Equal(t, []pb.ServerAddress{
+		pb.ServerAddress("111:1"),
+		pb.ServerAddress("111:2"),
+		pb.ServerAddress("111:3"),
+	}, c.filerLeaders.GetLeaders())
+
+	// remove oldest
+	c.RemoveClusterNode("filer", pb.ServerAddress("111:1"))
+	assert.Equal(t, []pb.ServerAddress{
+		pb.ServerAddress("111:6"),
+		pb.ServerAddress("111:2"),
+		pb.ServerAddress("111:3"),
+	}, c.filerLeaders.GetLeaders())
+
+	// remove oldest
+	c.RemoveClusterNode("filer", pb.ServerAddress("111:1"))
+
+}
@@ -120,7 +120,7 @@ func runBackup(cmd *Command, args []string) bool {
 	}

 	if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
-		if err = v.Compact2(30*1024*1024*1024, 0); err != nil {
+		if err = v.Compact2(30*1024*1024*1024, 0, nil); err != nil {
 			fmt.Printf("Compact Volume before synchronizing %v\n", err)
 			return true
 		}
@@ -50,7 +50,7 @@ func runCompact(cmd *Command, args []string) bool {
 			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	} else {
-		if err = v.Compact2(preallocate, 0); err != nil {
+		if err = v.Compact2(preallocate, 0, nil); err != nil {
 			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	}
@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"net/http"
 	"os"
-	"strings"
 	"time"

 	"google.golang.org/grpc/reflection"
@@ -46,7 +45,6 @@ type FilerOptions struct {
 	enableNotification      *bool
 	disableHttp             *bool
 	cipher                  *bool
-	peers                   *string
 	metricsHttpPort         *int
 	saveToFilerLimit        *int
 	defaultLevelDbDirectory *string
@@ -72,7 +70,6 @@ func init() {
 	f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
 	f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
 	f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
-	f.peers = cmdFiler.Flag.String("peers", "", "all filers sharing the same filer store in comma separated ip:port list")
 	f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
 	f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
@@ -186,11 +183,6 @@ func (fo *FilerOptions) startFiler() {

 	defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")

-	var peers []string
-	if *fo.peers != "" {
-		peers = strings.Split(*fo.peers, ",")
-	}
-
 	filerAddress := pb.NewServerAddress(*fo.ip, *fo.port, *fo.portGrpc)

 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
@@ -207,7 +199,6 @@ func (fo *FilerOptions) startFiler() {
 		Host:                  filerAddress,
 		Cipher:                *fo.cipher,
 		SaveToFilerLimit:      int64(*fo.saveToFilerLimit),
-		Filers:                pb.FromAddressStrings(peers),
 		ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
 	})
 	if nfs_err != nil {
@@ -30,6 +30,8 @@ var cmdFilerMetaTail = &Command{
 	weed filer.meta.tail -timeAgo=30h | jq .
 	weed filer.meta.tail -timeAgo=30h | jq .eventNotification.newEntry.name

+	weed filer.meta.tail -timeAgo=30h -es=http://<elasticSearchServerHost>:<port> -es.index=seaweedfs
+
  `,
 }

@@ -199,8 +199,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				return client.WriteDirectory(dest, message.NewEntry)
 			}
 			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
-			reader := filer.NewFileReader(filerSource, message.NewEntry)
-			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
+			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
 			if writeErr != nil {
 				return writeErr
 			}
@@ -264,9 +263,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 						return client.UpdateFileMetadata(oldDest, message.OldEntry, message.NewEntry)
 					} else {
 						newDest := toRemoteStorageLocation(newBucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), newRemoteStorageMountLocation)
-						reader := filer.NewFileReader(filerSource, message.NewEntry)
-						glog.V(0).Infof("create %s", remote_storage.FormatLocation(newDest))
-						remoteEntry, writeErr := client.WriteFile(newDest, message.NewEntry, reader)
+						remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, newDest)
 						if writeErr != nil {
 							return writeErr
 						}
@@ -303,9 +300,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				if message.NewEntry.IsDirectory {
 					return client.WriteDirectory(newDest, message.NewEntry)
 				}
-				reader := filer.NewFileReader(filerSource, message.NewEntry)
-				glog.V(0).Infof("create %s", remote_storage.FormatLocation(newDest))
-				remoteEntry, writeErr := client.WriteFile(newDest, message.NewEntry, reader)
+				remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, newDest)
 				if writeErr != nil {
 					return writeErr
 				}
@@ -3,6 +3,10 @@ package command
 import (
 	"context"
 	"fmt"
+	"os"
+	"strings"
+	"time"
+
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb"
@@ -13,9 +17,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
-	"os"
-	"strings"
-	"time"
 )

 func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string) error {
@@ -108,8 +109,7 @@ func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string,
 				return client.WriteDirectory(dest, message.NewEntry)
 			}
 			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
-			reader := filer.NewFileReader(filerSource, message.NewEntry)
-			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
+			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
 			if writeErr != nil {
 				return writeErr
 			}
@@ -146,9 +146,7 @@ func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string,
 			if err := client.DeleteFile(oldDest); err != nil {
 				return err
 			}
-			reader := filer.NewFileReader(filerSource, message.NewEntry)
-			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
-			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
+			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
 			if writeErr != nil {
 				return writeErr
 			}
@@ -160,6 +158,23 @@ func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string,
 	return eachEntryFunc, nil
 }

+func retriedWriteFile(client remote_storage.RemoteStorageClient, filerSource *source.FilerSource, newEntry *filer_pb.Entry, dest *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
+	var writeErr error
+	err = util.Retry("writeFile", func() error {
+		reader := filer.NewFileReader(filerSource, newEntry)
+		glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
+		remoteEntry, writeErr = client.WriteFile(dest, newEntry, reader)
+		if writeErr != nil {
+			return writeErr
+		}
+		return nil
+	})
+	if err != nil {
+		glog.Errorf("write to %s: %v", dest, err)
+	}
+	return
+}
+
 func collectLastSyncOffset(filerClient filer_pb.FilerClient, grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, mountedDir string, timeAgo time.Duration) time.Time {
 	// 1. specified by timeAgo
 	// 2. last offset timestamp for this directory
@@ -61,12 +61,12 @@ connection_max_lifetime_seconds = 0
 interpolateParams = false
 # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
 enableUpsert = true
-upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
+upsertQuery = """INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""

 [mysql2]  # or memsql, tidb
 enabled = false
 createTable = """
-  CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
+  CREATE TABLE IF NOT EXISTS `%s` (
    dirhash BIGINT,
    name VARCHAR(1000) BINARY,
    directory TEXT BINARY,
@@ -85,7 +85,7 @@ connection_max_lifetime_seconds = 0
 interpolateParams = false
 # if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
 enableUpsert = true
-upsertQuery = """INSERT INTO ` + "`%s`" + ` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""
+upsertQuery = """INSERT INTO `%s` (dirhash,name,directory,meta) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE meta = VALUES(meta)"""

 [postgres] # or cockroachdb, YugabyteDB
 # CREATE TABLE IF NOT EXISTS filemeta (
@@ -167,6 +167,14 @@ database = 0
 # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
 superLargeDirectories = []

+[redis2_sentinel]
+enabled = false
+addresses = ["172.22.12.7:26379","172.22.12.8:26379","172.22.12.9:26379"]
+masterName = "master"
+username = ""
+password = ""
+database = 0
+
 [redis_cluster2]
 enabled = false
 addresses = [
@@ -14,6 +14,7 @@ scripts = """
  volume.deleteEmpty -quietFor=24h -force
  volume.balance -force
  volume.fix.replication
+  s3.clean.uploads -timeAgo=24h
  unlock
 """
 sleep_minutes = 17          # sleep minutes between each script execution
@@ -106,7 +106,6 @@ func init() {
 	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 4, "split files larger than the limit")
 	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
 	filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers")
-	filerOptions.peers = cmdServer.Flag.String("filer.peers", "", "all filers sharing the same filer store in comma separated ip:port list")
 	filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.")
 	filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size")

@@ -37,7 +37,7 @@ func runShell(command *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	shellOptions.GrpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")

-	if *shellOptions.Masters == "" && *shellInitialFiler == "" {
+	if *shellOptions.Masters == "" {
 		util.LoadConfiguration("shell", false)
 		v := util.GetViper()
 		cluster := v.GetString("cluster.default")
@@ -45,15 +45,14 @@ func runShell(command *Command, args []string) bool {
 			cluster = *shellCluster
 		}
 		if cluster == "" {
-			*shellOptions.Masters, *shellInitialFiler = "localhost:9333", "localhost:8888"
+			*shellOptions.Masters = "localhost:9333"
 		} else {
 			*shellOptions.Masters = v.GetString("cluster." + cluster + ".master")
 			*shellInitialFiler = v.GetString("cluster." + cluster + ".filer")
+			fmt.Printf("master: %s filer: %s\n", *shellOptions.Masters, *shellInitialFiler)
 		}
 	}

-	fmt.Printf("master: %s filer: %s\n", *shellOptions.Masters, *shellInitialFiler)
-
 	shellOptions.FilerAddress = pb.ServerAddress(*shellInitialFiler)
 	shellOptions.Directory = "/"

@@ -40,7 +40,7 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterva
 	for _, point := range points {
 		if point.isStart {
 			if len(queue) > 0 {
-				lastIndex := len(queue) -1
+				lastIndex := len(queue) - 1
 				lastPoint := queue[lastIndex]
 				if point.x != prevX && lastPoint.ts < point.ts {
 					visibles = addToVisibles(visibles, prevX, lastPoint, point)
@@ -52,7 +52,7 @@ func TestReadResolvedChunks(t *testing.T) {

 func TestRandomizedReadResolvedChunks(t *testing.T) {

-	var limit int64 = 1024*1024
+	var limit int64 = 1024 * 1024
 	array := make([]int64, limit)
 	var chunks []*filer_pb.FileChunk
 	for ts := int64(0); ts < 1024; ts++ {
@@ -75,7 +75,7 @@ func TestRandomizedReadResolvedChunks(t *testing.T) {
 	visibles := readResolvedChunks(chunks)

 	for _, visible := range visibles {
-		for i := visible.start; i<visible.stop;i++{
+		for i := visible.start; i < visible.stop; i++ {
 			if array[i] != visible.modifiedTime {
 				t.Errorf("position %d expected ts %d actual ts %d", i, array[i], visible.modifiedTime)
 			}
@@ -101,12 +101,12 @@ func randomWrite(array []int64, start int64, size int64, ts int64) *filer_pb.Fil

 func TestSequentialReadResolvedChunks(t *testing.T) {

-	var chunkSize int64 = 1024*1024*2
+	var chunkSize int64 = 1024 * 1024 * 2
 	var chunks []*filer_pb.FileChunk
 	for ts := int64(0); ts < 13; ts++ {
 		chunks = append(chunks, &filer_pb.FileChunk{
 			FileId: "",
-			Offset: chunkSize*ts,
+			Offset: chunkSize * ts,
 			Size:   uint64(chunkSize),
 			Mtime:  1,
 		})
@@ -3,7 +3,9 @@ package filer
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/cluster"
 	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"os"
 	"strings"
 	"time"
@@ -50,7 +52,7 @@ type Filer struct {
 func NewFiler(masters []pb.ServerAddress, grpcDialOption grpc.DialOption,
 	filerHost pb.ServerAddress, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
 	f := &Filer{
-		MasterClient:        wdclient.NewMasterClient(grpcDialOption, "filer", filerHost, dataCenter, masters),
+		MasterClient:        wdclient.NewMasterClient(grpcDialOption, cluster.FilerType, filerHost, dataCenter, masters),
 		fileIdDeletionQueue: util.NewUnboundedQueue(),
 		GrpcDialOption:      grpcDialOption,
 		FilerConf:           NewFilerConf(),
@@ -66,22 +68,38 @@ func NewFiler(masters []pb.ServerAddress, grpcDialOption grpc.DialOption,
 	return f
 }

-func (f *Filer) AggregateFromPeers(self pb.ServerAddress, filers []pb.ServerAddress) {
+func (f *Filer) AggregateFromPeers(self pb.ServerAddress) {

-	// set peers
-	found := false
-	for _, peer := range filers {
-		if peer == self {
-			found = true
+	f.MetaAggregator = NewMetaAggregator(f, self, f.GrpcDialOption)
+	f.MasterClient.OnPeerUpdate = f.MetaAggregator.OnPeerUpdate
+
+	for _, peerUpdate := range f.ListExistingPeerUpdates() {
+		f.MetaAggregator.OnPeerUpdate(peerUpdate)
+	}
+
+}
+
+func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNodeUpdate){
+
+	if grpcErr := pb.WithMasterClient(f.MasterClient.GetMaster(), f.GrpcDialOption, func(client master_pb.SeaweedClient) error {
+		resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
+			ClientType: cluster.FilerType,
+		})
+
+		glog.V(0).Infof("the cluster has %d filers\n", len(resp.ClusterNodes))
+		for _, node := range resp.ClusterNodes {
+			existingNodes = append(existingNodes, &master_pb.ClusterNodeUpdate{
+				NodeType: cluster.FilerType,
+				Address:  node.Address,
+				IsLeader: node.IsLeader,
+				IsAdd:    true,
+			})
 		}
+		return err
+	}); grpcErr != nil {
+		glog.V(0).Infof("connect to %s: %v", f.MasterClient.GetMaster(), grpcErr)
 	}
-	if !found {
-		filers = append(filers, self)
-	}
-
-	f.MetaAggregator = NewMetaAggregator(filers, f.GrpcDialOption)
-	f.MetaAggregator.StartLoopSubscribe(f, self)
-
+	return
 }

 func (f *Filer) SetStore(store FilerStore) {
@@ -117,7 +135,7 @@ func (fs *Filer) GetMaster() pb.ServerAddress {
 	return fs.MasterClient.GetMaster()
 }

-func (fs *Filer) KeepConnectedToMaster() {
+func (fs *Filer) KeepMasterClientConnected() {
 	fs.MasterClient.KeepConnectedToMaster()
 }

@@ -15,6 +15,9 @@ const (
 	MsgFailDelNonEmptyFolder = "fail to delete non-empty folder"
 )

+type OnChunksFunc func([]*filer_pb.FileChunk) error
+type OnHardLinkIdsFunc func([]HardLinkId) error
+
 func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32) (err error) {
 	if p == "/" {
 		return nil
@@ -27,20 +30,29 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR

 	isDeleteCollection := f.isBucket(entry)

-	var chunks []*filer_pb.FileChunk
-	var hardLinkIds []HardLinkId
-	chunks = append(chunks, entry.Chunks...)
 	if entry.IsDirectory() {
 		// delete the folder children, not including the folder itself
-		var dirChunks []*filer_pb.FileChunk
-		var dirHardLinkIds []HardLinkId
-		dirChunks, dirHardLinkIds, err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isDeleteCollection, isFromOtherCluster, signatures)
+		err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isDeleteCollection, isFromOtherCluster, signatures, func(chunks []*filer_pb.FileChunk) error {
+			if shouldDeleteChunks && !isDeleteCollection {
+				f.DirectDeleteChunks(chunks)
+			}
+			return nil
+		}, func(hardLinkIds []HardLinkId) error {
+			// A case not handled:
+			// what if the chunk is in a different collection?
+			if shouldDeleteChunks {
+				f.maybeDeleteHardLinks(hardLinkIds)
+			}
+			return nil
+		})
 		if err != nil {
 			glog.V(0).Infof("delete directory %s: %v", p, err)
 			return fmt.Errorf("delete directory %s: %v", p, err)
 		}
-		chunks = append(chunks, dirChunks...)
-		hardLinkIds = append(hardLinkIds, dirHardLinkIds...)
+	}
+
+	if shouldDeleteChunks && !isDeleteCollection {
+		f.DirectDeleteChunks(entry.Chunks)
 	}

 	// delete the file or folder
@@ -49,15 +61,6 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 		return fmt.Errorf("delete file %s: %v", p, err)
 	}

-	if shouldDeleteChunks && !isDeleteCollection {
-		f.DirectDeleteChunks(chunks)
-	}
-	// A case not handled:
-	// what if the chunk is in a different collection?
-	if shouldDeleteChunks {
-		f.maybeDeleteHardLinks(hardLinkIds)
-	}
-
 	if isDeleteCollection {
 		collectionName := entry.Name()
 		f.doDeleteCollection(collectionName)
@@ -67,7 +70,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 	return nil
 }

-func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isDeletingBucket, isFromOtherCluster bool, signatures []int32) (chunks []*filer_pb.FileChunk, hardlinkIds []HardLinkId, err error) {
+func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isDeletingBucket, isFromOtherCluster bool, signatures []int32, onChunksFn OnChunksFunc, onHardLinkIdsFn OnHardLinkIdsFunc) (err error) {

 	lastFileName := ""
 	includeLastFile := false
@@ -76,34 +79,30 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 			entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "", "")
 			if err != nil {
 				glog.Errorf("list folder %s: %v", entry.FullPath, err)
-				return nil, nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err)
+				return fmt.Errorf("list folder %s: %v", entry.FullPath, err)
 			}
 			if lastFileName == "" && !isRecursive && len(entries) > 0 {
 				// only for first iteration in the loop
-				glog.Errorf("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name())
-				return nil, nil, fmt.Errorf("%s: %s", MsgFailDelNonEmptyFolder, entry.FullPath)
+				glog.V(0).Infof("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name())
+				return fmt.Errorf("%s: %s", MsgFailDelNonEmptyFolder, entry.FullPath)
 			}

 			for _, sub := range entries {
 				lastFileName = sub.Name()
-				var dirChunks []*filer_pb.FileChunk
-				var dirHardLinkIds []HardLinkId
 				if sub.IsDirectory() {
 					subIsDeletingBucket := f.isBucket(sub)
-					dirChunks, dirHardLinkIds, err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, false, nil)
-					chunks = append(chunks, dirChunks...)
-					hardlinkIds = append(hardlinkIds, dirHardLinkIds...)
+					err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, false, nil, onChunksFn, onHardLinkIdsFn)
 				} else {
 					f.NotifyUpdateEvent(ctx, sub, nil, shouldDeleteChunks, isFromOtherCluster, nil)
 					if len(sub.HardLinkId) != 0 {
 						// hard link chunk data are deleted separately
-						hardlinkIds = append(hardlinkIds, sub.HardLinkId)
+						err = onHardLinkIdsFn([]HardLinkId{sub.HardLinkId})
 					} else {
-						chunks = append(chunks, sub.Chunks...)
+						err = onChunksFn(sub.Chunks)
 					}
 				}
 				if err != nil && !ignoreRecursiveError {
-					return nil, nil, err
+					return err
 				}
 			}

@@ -113,15 +112,15 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 		}
 	}

-	glog.V(3).Infof("deleting directory %v delete %d chunks: %v", entry.FullPath, len(chunks), shouldDeleteChunks)
+	glog.V(3).Infof("deleting directory %v delete chunks: %v", entry.FullPath, shouldDeleteChunks)

 	if storeDeletionErr := f.Store.DeleteFolderChildren(ctx, entry.FullPath); storeDeletionErr != nil {
-		return nil, nil, fmt.Errorf("filer store delete: %v", storeDeletionErr)
+		return fmt.Errorf("filer store delete: %v", storeDeletionErr)
 	}

 	f.NotifyUpdateEvent(ctx, entry, nil, shouldDeleteChunks, isFromOtherCluster, signatures)

-	return chunks, hardlinkIds, nil
+	return nil
 }

 func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shouldDeleteChunks bool, isFromOtherCluster bool, signatures []int32) (err error) {
@@ -136,6 +136,7 @@ func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
 	}
 	if newEntry == nil {
 		f.DeleteChunks(oldEntry.Chunks)
+		return
 	}

 	var toDelete []*filer_pb.FileChunk
@@ -296,7 +296,7 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u
 		}
 		if count < limit {
 			notPrefixed = notPrefixed[:0]
-			_, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool {
+			lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool {
 				notPrefixed = append(notPrefixed, entry)
 				return true
 			})
@@ -3,6 +3,8 @@ package filer
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/cluster"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"io"
 	"sync"
@@ -18,9 +20,13 @@ import (
 )

 type MetaAggregator struct {
-	filers         []pb.ServerAddress
-	grpcDialOption grpc.DialOption
-	MetaLogBuffer  *log_buffer.LogBuffer
+	filer           *Filer
+	self            pb.ServerAddress
+	isLeader        bool
+	grpcDialOption  grpc.DialOption
+	MetaLogBuffer   *log_buffer.LogBuffer
+	peerStatues     map[pb.ServerAddress]struct{}
+	peerStatuesLock sync.Mutex
 	// notifying clients
 	ListenersLock sync.Mutex
 	ListenersCond *sync.Cond
@@ -28,10 +34,12 @@ type MetaAggregator struct {

 // MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
 // The old data comes from what each LocalMetadata persisted on disk.
-func NewMetaAggregator(filers []pb.ServerAddress, grpcDialOption grpc.DialOption) *MetaAggregator {
+func NewMetaAggregator(filer *Filer, self pb.ServerAddress, grpcDialOption grpc.DialOption) *MetaAggregator {
 	t := &MetaAggregator{
-		filers:         filers,
+		filer:          filer,
+		self:           self,
 		grpcDialOption: grpcDialOption,
+		peerStatues:    make(map[pb.ServerAddress]struct{}),
 	}
 	t.ListenersCond = sync.NewCond(&t.ListenersLock)
 	t.MetaLogBuffer = log_buffer.NewLogBuffer("aggr", LogFlushInterval, nil, func() {
@@ -40,10 +48,35 @@ func NewMetaAggregator(filers []pb.ServerAddress, grpcDialOption grpc.DialOption
 	return t
 }

-func (ma *MetaAggregator) StartLoopSubscribe(f *Filer, self pb.ServerAddress) {
-	for _, filer := range ma.filers {
-		go ma.subscribeToOneFiler(f, self, filer)
+func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate) {
+	if update.NodeType != cluster.FilerType {
+		return
 	}
+
+	address := pb.ServerAddress(update.Address)
+	if update.IsAdd {
+		// every filer should subscribe to a new filer
+		ma.setActive(address, true)
+		go ma.subscribeToOneFiler(ma.filer, ma.self, address)
+	} else {
+		ma.setActive(address, false)
+	}
+}
+
+func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) {
+	ma.peerStatuesLock.Lock()
+	defer ma.peerStatuesLock.Unlock()
+	if isActive {
+		ma.peerStatues[address] = struct{}{}
+	} else {
+		delete(ma.peerStatues, address)
+	}
+}
+func (ma *MetaAggregator) isActive(address pb.ServerAddress)(isActive bool) {
+	ma.peerStatuesLock.Lock()
+	defer ma.peerStatuesLock.Unlock()
+	_, isActive = ma.peerStatues[address]
+	return
 }

 func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress) {
@@ -149,6 +182,10 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self pb.ServerAddress, p

 			}
 		})
+		if !ma.isActive(peer) {
+			glog.V(0).Infof("stop subscribing remote %s meta change", peer)
+			return
+		}
 		if err != nil {
 			glog.V(0).Infof("subscribing remote %s meta change: %v", peer, err)
 			time.Sleep(1733 * time.Millisecond)
@@ -25,9 +25,9 @@ func MapRemoteStorageLocationPathToFullPath(localMountedDir util.FullPath, remot
 	return localMountedDir.Child(remoteLocationPath[len(remoteMountedLocation.Path):])
 }

-func DownloadToLocal(filerClient filer_pb.FilerClient, remoteConf *remote_pb.RemoteConf, remoteLocation *remote_pb.RemoteStorageLocation, parent util.FullPath, entry *filer_pb.Entry) error {
+func CacheRemoteObjectToLocalCluster(filerClient filer_pb.FilerClient, remoteConf *remote_pb.RemoteConf, remoteLocation *remote_pb.RemoteStorageLocation, parent util.FullPath, entry *filer_pb.Entry) error {
 	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		_, err := client.DownloadToLocal(context.Background(), &filer_pb.DownloadToLocalRequest{
+		_, err := client.CacheRemoteObjectToLocalCluster(context.Background(), &filer_pb.CacheRemoteObjectToLocalClusterRequest{
 			Directory: string(parent),
 			Name:      entry.Name,
 		})
@@ -0,0 +1,45 @@
+package redis2
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/go-redis/redis/v8"
+	"time"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &Redis2SentinelStore{})
+}
+
+type Redis2SentinelStore struct {
+	UniversalRedis2Store
+}
+
+func (store *Redis2SentinelStore) GetName() string {
+	return "redis2_sentinel"
+}
+
+func (store *Redis2SentinelStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	return store.initialize(
+		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"masterName"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetInt(prefix+"database"),
+	)
+}
+
+func (store *Redis2SentinelStore) initialize(addresses []string, masterName string, username string, password string, database int) (err error) {
+	store.Client = redis.NewFailoverClient(&redis.FailoverOptions{
+		MasterName:      masterName,
+		SentinelAddrs:   addresses,
+		Username:        username,
+		Password:        password,
+		DB:              database,
+		MinRetryBackoff: time.Millisecond * 100,
+		MaxRetryBackoff: time.Minute * 1,
+		ReadTimeout:     time.Second * 30,
+		WriteTimeout:    time.Second * 5,
+	})
+	return
+}
@@ -133,7 +133,10 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful
 		return nil
 	}

-	members, err := store.Client.ZRange(ctx, genDirectoryListKey(string(fullpath)), 0, -1).Result()
+	members, err := store.Client.ZRangeByLex(ctx, genDirectoryListKey(string(fullpath)), &redis.ZRangeBy{
+		Min: "-",
+		Max: "+",
+	}).Result()
 	if err != nil {
 		return fmt.Errorf("DeleteFolderChildren %s : %v", fullpath, err)
 	}
@@ -158,14 +161,22 @@ func (store *UniversalRedis2Store) ListDirectoryPrefixedEntries(ctx context.Cont
 func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {

 	dirListKey := genDirectoryListKey(string(dirPath))
-	start := int64(0)
+
+	min := "-"
 	if startFileName != "" {
-		start, _ = store.Client.ZRank(ctx, dirListKey, startFileName).Result()
-		if !includeStartFile {
-			start++
+		if includeStartFile {
+			min = "[" + startFileName
+		} else {
+			min = "(" + startFileName
 		}
 	}
-	members, err := store.Client.ZRange(ctx, dirListKey, start, start+int64(limit)-1).Result()
+
+	members, err := store.Client.ZRangeByLex(ctx, dirListKey, &redis.ZRangeBy{
+		Min:    min,
+		Max:    "+",
+		Offset: 0,
+		Count:  limit,
+	}).Result()
 	if err != nil {
 		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
 	}
@@ -297,7 +297,7 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
 	var buffer bytes.Buffer
 	var shouldRetry bool
 	for _, urlString := range urlStrings {
-		shouldRetry, err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
+		shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
 			buffer.Write(data)
 		})
 		if !shouldRetry {
@@ -67,7 +67,6 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 	return nil
 }

-
 func (dir *Dir) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamRenameEntryResponse) error {
 	// comes from filer StreamRenameEntry, can only be create or delete entry

@@ -119,7 +118,7 @@ func (dir *Dir) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamR
 		}
 		dir.wfs.handlesLock.Unlock()

-	}else if resp.EventNotification.OldEntry != nil {
+	} else if resp.EventNotification.OldEntry != nil {
 		// without new entry, only old entry name exists. This is the second step to delete old entry
 		if err := dir.wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, util.NewFullPath(resp.Directory, resp.EventNotification.OldEntry.Name), nil); err != nil {
 			return err
@@ -364,15 +364,15 @@ func (file *File) getEntry() *filer_pb.Entry {
 func (file *File) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entry, error) {
 	err := file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

-		request := &filer_pb.DownloadToLocalRequest{
+		request := &filer_pb.CacheRemoteObjectToLocalClusterRequest{
 			Directory: file.dir.FullPath(),
 			Name:      entry.Name,
 		}

 		glog.V(4).Infof("download entry: %v", request)
-		resp, err := client.DownloadToLocal(context.Background(), request)
+		resp, err := client.CacheRemoteObjectToLocalCluster(context.Background(), request)
 		if err != nil {
-			glog.Errorf("DownloadToLocal file %s/%s: %v", file.dir.FullPath(), file.Name, err)
+			glog.Errorf("CacheRemoteObjectToLocalCluster file %s/%s: %v", file.dir.FullPath(), file.Name, err)
 			return fuse.EIO
 		}

@@ -8,7 +8,7 @@ import (
 	"net/http"
 )

-func writeIamErrorResponse(w http.ResponseWriter, err error, object string, value string, msg error) {
+func writeIamErrorResponse(w http.ResponseWriter, r *http.Request, err error, object string, value string, msg error) {
 	errCode := err.Error()
 	errorResp := ErrorResponse{}
 	errorResp.Error.Type = "Sender"
@@ -22,10 +22,10 @@ func writeIamErrorResponse(w http.ResponseWriter, err error, object string, valu
 	case iam.ErrCodeNoSuchEntityException:
 		msg := fmt.Sprintf("The %s with name %s cannot be found.", object, value)
 		errorResp.Error.Message = &msg
-		s3err.WriteXMLResponse(w, http.StatusNotFound, errorResp)
+		s3err.WriteXMLResponse(w, r, http.StatusNotFound, errorResp)
 	case iam.ErrCodeServiceFailureException:
-		s3err.WriteXMLResponse(w, http.StatusInternalServerError, errorResp)
+		s3err.WriteXMLResponse(w, r, http.StatusInternalServerError, errorResp)
 	default:
-		s3err.WriteXMLResponse(w, http.StatusInternalServerError, errorResp)
+		s3err.WriteXMLResponse(w, r, http.StatusInternalServerError, errorResp)
 	}
 }
@@ -362,7 +362,7 @@ func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu

 func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 	if err := r.ParseForm(); err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
 		return
 	}
 	values := r.PostForm
@@ -370,7 +370,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 	s3cfgLock.RLock()
 	s3cfg := &iam_pb.S3ApiConfiguration{}
 	if err := iama.s3ApiConfig.GetS3ApiConfiguration(s3cfg); err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 	s3cfgLock.RUnlock()
@@ -392,7 +392,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		userName := values.Get("UserName")
 		response, err = iama.GetUser(s3cfg, userName)
 		if err != nil {
-			writeIamErrorResponse(w, err, "user", userName, nil)
+			writeIamErrorResponse(w, r, err, "user", userName, nil)
 			return
 		}
 		changed = false
@@ -400,7 +400,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		userName := values.Get("UserName")
 		response, err = iama.DeleteUser(s3cfg, userName)
 		if err != nil {
-			writeIamErrorResponse(w, err, "user", userName, nil)
+			writeIamErrorResponse(w, r, err, "user", userName, nil)
 			return
 		}
 	case "CreateAccessKey":
@@ -411,33 +411,33 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		response, err = iama.CreatePolicy(s3cfg, values)
 		if err != nil {
 			glog.Errorf("CreatePolicy:  %+v", err)
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
 			return
 		}
 	case "PutUserPolicy":
 		response, err = iama.PutUserPolicy(s3cfg, values)
 		if err != nil {
 			glog.Errorf("PutUserPolicy:  %+v", err)
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
 			return
 		}
 	case "GetUserPolicy":
 		response, err = iama.GetUserPolicy(s3cfg, values)
 		if err != nil {
-			writeIamErrorResponse(w, err, "user", values.Get("UserName"), nil)
+			writeIamErrorResponse(w, r, err, "user", values.Get("UserName"), nil)
 			return
 		}
 		changed = false
 	case "DeleteUserPolicy":
 		if response, err = iama.DeleteUserPolicy(s3cfg, values); err != nil {
-			writeIamErrorResponse(w, err, "user", values.Get("UserName"), nil)
+			writeIamErrorResponse(w, r, err, "user", values.Get("UserName"), nil)
 		}
 	default:
 		errNotImplemented := s3err.GetAPIError(s3err.ErrNotImplemented)
 		errorResponse := ErrorResponse{}
 		errorResponse.Error.Code = &errNotImplemented.Code
 		errorResponse.Error.Message = &errNotImplemented.Description
-		s3err.WriteXMLResponse(w, errNotImplemented.HTTPStatusCode, errorResponse)
+		s3err.WriteXMLResponse(w, r, errNotImplemented.HTTPStatusCode, errorResponse)
 		return
 	}
 	if changed {
@@ -445,9 +445,9 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		err := iama.s3ApiConfig.PutS3ApiConfiguration(s3cfg)
 		s3cfgLock.Unlock()
 		if err != nil {
-			writeIamErrorResponse(w, fmt.Errorf(iam.ErrCodeServiceFailureException), "", "", err)
+			writeIamErrorResponse(w, r, fmt.Errorf(iam.ErrCodeServiceFailureException), "", "", err)
 			return
 		}
 	}
-	s3err.WriteXMLResponse(w, http.StatusOK, response)
+	s3err.WriteXMLResponse(w, r, http.StatusOK, response)
 }
@@ -49,7 +49,7 @@ var s3ApiConfigure IamS3ApiConfig
 func NewIamApiServer(router *mux.Router, option *IamServerOption) (iamApiServer *IamApiServer, err error) {
 	s3ApiConfigure = IamS3ApiConfigure{
 		option:       option,
-		masterClient: wdclient.NewMasterClient(option.GrpcDialOption, pb.AdminShellClient, "", "", option.Masters),
+		masterClient: wdclient.NewMasterClient(option.GrpcDialOption, "iam", "", "", option.Masters),
 	}
 	s3Option := s3api.S3ApiServerOption{Filer: option.Filer}
 	iamApiServer = &IamApiServer{
@@ -3,6 +3,7 @@ package broker
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/cluster"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"time"

@@ -93,14 +94,16 @@ func (broker *MessageBroker) checkFilers() {
 	for !found {
 		for _, master := range masters {
 			err := broker.withMasterClient(master, func(client master_pb.SeaweedClient) error {
-				resp, err := client.ListMasterClients(context.Background(), &master_pb.ListMasterClientsRequest{
-					ClientType: "filer",
+				resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
+					ClientType: cluster.FilerType,
 				})
 				if err != nil {
 					return err
 				}

-				filers = append(filers, pb.FromAddressStrings(resp.GrpcAddresses)...)
+				for _, clusterNode := range resp.ClusterNodes {
+					filers = append(filers, pb.ServerAddress(clusterNode.Address))
+				}

 				return nil
 			})
@@ -69,7 +69,7 @@ service SeaweedFiler {
    rpc KvPut (KvPutRequest) returns (KvPutResponse) {
    }

-    rpc DownloadToLocal (DownloadToLocalRequest) returns (DownloadToLocalResponse) {
+    rpc CacheRemoteObjectToLocalCluster (CacheRemoteObjectToLocalClusterRequest) returns (CacheRemoteObjectToLocalClusterResponse) {
    }
 }

@@ -403,10 +403,10 @@ message FilerConf {
 /////////////////////////
 // Remote Storage related
 /////////////////////////
-message DownloadToLocalRequest {
+message CacheRemoteObjectToLocalClusterRequest {
    string directory = 1;
    string name = 2;
 }
-message DownloadToLocalResponse {
+message CacheRemoteObjectToLocalClusterResponse {
    Entry entry = 1;
 }
@@ -3334,7 +3334,7 @@ func (x *FilerConf) GetLocations() []*FilerConf_PathConf {
 /////////////////////////
 // Remote Storage related
 /////////////////////////
-type DownloadToLocalRequest struct {
+type CacheRemoteObjectToLocalClusterRequest struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
@@ -3343,8 +3343,8 @@ type DownloadToLocalRequest struct {
 	Name      string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
 }

-func (x *DownloadToLocalRequest) Reset() {
-	*x = DownloadToLocalRequest{}
+func (x *CacheRemoteObjectToLocalClusterRequest) Reset() {
+	*x = CacheRemoteObjectToLocalClusterRequest{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_filer_proto_msgTypes[51]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -3352,13 +3352,13 @@ func (x *DownloadToLocalRequest) Reset() {
 	}
 }

-func (x *DownloadToLocalRequest) String() string {
+func (x *CacheRemoteObjectToLocalClusterRequest) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }

-func (*DownloadToLocalRequest) ProtoMessage() {}
+func (*CacheRemoteObjectToLocalClusterRequest) ProtoMessage() {}

-func (x *DownloadToLocalRequest) ProtoReflect() protoreflect.Message {
+func (x *CacheRemoteObjectToLocalClusterRequest) ProtoReflect() protoreflect.Message {
 	mi := &file_filer_proto_msgTypes[51]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -3370,26 +3370,26 @@ func (x *DownloadToLocalRequest) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }

-// Deprecated: Use DownloadToLocalRequest.ProtoReflect.Descriptor instead.
-func (*DownloadToLocalRequest) Descriptor() ([]byte, []int) {
+// Deprecated: Use CacheRemoteObjectToLocalClusterRequest.ProtoReflect.Descriptor instead.
+func (*CacheRemoteObjectToLocalClusterRequest) Descriptor() ([]byte, []int) {
 	return file_filer_proto_rawDescGZIP(), []int{51}
 }

-func (x *DownloadToLocalRequest) GetDirectory() string {
+func (x *CacheRemoteObjectToLocalClusterRequest) GetDirectory() string {
 	if x != nil {
 		return x.Directory
 	}
 	return ""
 }

-func (x *DownloadToLocalRequest) GetName() string {
+func (x *CacheRemoteObjectToLocalClusterRequest) GetName() string {
 	if x != nil {
 		return x.Name
 	}
 	return ""
 }

-type DownloadToLocalResponse struct {
+type CacheRemoteObjectToLocalClusterResponse struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
@@ -3397,8 +3397,8 @@ type DownloadToLocalResponse struct {
 	Entry *Entry `protobuf:"bytes,1,opt,name=entry,proto3" json:"entry,omitempty"`
 }

-func (x *DownloadToLocalResponse) Reset() {
-	*x = DownloadToLocalResponse{}
+func (x *CacheRemoteObjectToLocalClusterResponse) Reset() {
+	*x = CacheRemoteObjectToLocalClusterResponse{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_filer_proto_msgTypes[52]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -3406,13 +3406,13 @@ func (x *DownloadToLocalResponse) Reset() {
 	}
 }

-func (x *DownloadToLocalResponse) String() string {
+func (x *CacheRemoteObjectToLocalClusterResponse) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }

-func (*DownloadToLocalResponse) ProtoMessage() {}
+func (*CacheRemoteObjectToLocalClusterResponse) ProtoMessage() {}

-func (x *DownloadToLocalResponse) ProtoReflect() protoreflect.Message {
+func (x *CacheRemoteObjectToLocalClusterResponse) ProtoReflect() protoreflect.Message {
 	mi := &file_filer_proto_msgTypes[52]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -3424,12 +3424,12 @@ func (x *DownloadToLocalResponse) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }

-// Deprecated: Use DownloadToLocalResponse.ProtoReflect.Descriptor instead.
-func (*DownloadToLocalResponse) Descriptor() ([]byte, []int) {
+// Deprecated: Use CacheRemoteObjectToLocalClusterResponse.ProtoReflect.Descriptor instead.
+func (*CacheRemoteObjectToLocalClusterResponse) Descriptor() ([]byte, []int) {
 	return file_filer_proto_rawDescGZIP(), []int{52}
 }

-func (x *DownloadToLocalResponse) GetEntry() *Entry {
+func (x *CacheRemoteObjectToLocalClusterResponse) GetEntry() *Entry {
 	if x != nil {
 		return x.Entry
 	}
@@ -4042,16 +4042,18 @@ var file_filer_proto_rawDesc = []byte{
 	0x68, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11, 0x76,
 	0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x47, 0x72, 0x6f, 0x77, 0x74, 0x68, 0x43, 0x6f, 0x75, 0x6e, 0x74,
 	0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x08, 0x20,
-	0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x22, 0x4a, 0x0a,
-	0x16, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c,
+	0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x22, 0x5a, 0x0a,
+	0x26, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65,
+	0x63, 0x74, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72,
 	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63,
 	0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65,
 	0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20,
-	0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x40, 0x0a, 0x17, 0x44, 0x6f, 0x77,
-	0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x70,
+	0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x50, 0x0a, 0x27, 0x43, 0x61, 0x63,
+	0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x54, 0x6f,
+	0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70,
 	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x01, 0x20,
 	0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45,
-	0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x32, 0x98, 0x0e, 0x0a, 0x0c,
+	0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x32, 0xc9, 0x0e, 0x0a, 0x0c,
 	0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x12, 0x67, 0x0a, 0x14,
 	0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45,
 	0x6e, 0x74, 0x72, 0x79, 0x12, 0x25, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
@@ -4159,18 +4161,21 @@ var file_filer_proto_rawDesc = []byte{
 	0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x12, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65,
 	0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
 	0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x50,
-	0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x0f,
-	0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x12,
-	0x20, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x6c,
-	0x6f, 0x61, 0x64, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x1a, 0x21, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x6f, 0x77,
-	0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4f, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65,
-	0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65,
-	0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
-	0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61,
-	0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x66,
-	0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x88, 0x01, 0x0a,
+	0x1f, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65,
+	0x63, 0x74, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72,
+	0x12, 0x30, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x61, 0x63, 0x68,
+	0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x54, 0x6f, 0x4c,
+	0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65,
+	0x73, 0x74, 0x1a, 0x31, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x61,
+	0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x54,
+	0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x4f, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65,
+	0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c,
+	0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
+	0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65,
+	0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f,
+	0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }

 var (
@@ -4187,59 +4192,59 @@ func file_filer_proto_rawDescGZIP() []byte {

 var file_filer_proto_msgTypes = make([]protoimpl.MessageInfo, 57)
 var file_filer_proto_goTypes = []interface{}{
-	(*LookupDirectoryEntryRequest)(nil),   // 0: filer_pb.LookupDirectoryEntryRequest
-	(*LookupDirectoryEntryResponse)(nil),  // 1: filer_pb.LookupDirectoryEntryResponse
-	(*ListEntriesRequest)(nil),            // 2: filer_pb.ListEntriesRequest
-	(*ListEntriesResponse)(nil),           // 3: filer_pb.ListEntriesResponse
-	(*RemoteEntry)(nil),                   // 4: filer_pb.RemoteEntry
-	(*Entry)(nil),                         // 5: filer_pb.Entry
-	(*FullEntry)(nil),                     // 6: filer_pb.FullEntry
-	(*EventNotification)(nil),             // 7: filer_pb.EventNotification
-	(*FileChunk)(nil),                     // 8: filer_pb.FileChunk
-	(*FileChunkManifest)(nil),             // 9: filer_pb.FileChunkManifest
-	(*FileId)(nil),                        // 10: filer_pb.FileId
-	(*FuseAttributes)(nil),                // 11: filer_pb.FuseAttributes
-	(*CreateEntryRequest)(nil),            // 12: filer_pb.CreateEntryRequest
-	(*CreateEntryResponse)(nil),           // 13: filer_pb.CreateEntryResponse
-	(*UpdateEntryRequest)(nil),            // 14: filer_pb.UpdateEntryRequest
-	(*UpdateEntryResponse)(nil),           // 15: filer_pb.UpdateEntryResponse
-	(*AppendToEntryRequest)(nil),          // 16: filer_pb.AppendToEntryRequest
-	(*AppendToEntryResponse)(nil),         // 17: filer_pb.AppendToEntryResponse
-	(*DeleteEntryRequest)(nil),            // 18: filer_pb.DeleteEntryRequest
-	(*DeleteEntryResponse)(nil),           // 19: filer_pb.DeleteEntryResponse
-	(*AtomicRenameEntryRequest)(nil),      // 20: filer_pb.AtomicRenameEntryRequest
-	(*AtomicRenameEntryResponse)(nil),     // 21: filer_pb.AtomicRenameEntryResponse
-	(*StreamRenameEntryRequest)(nil),      // 22: filer_pb.StreamRenameEntryRequest
-	(*StreamRenameEntryResponse)(nil),     // 23: filer_pb.StreamRenameEntryResponse
-	(*AssignVolumeRequest)(nil),           // 24: filer_pb.AssignVolumeRequest
-	(*AssignVolumeResponse)(nil),          // 25: filer_pb.AssignVolumeResponse
-	(*LookupVolumeRequest)(nil),           // 26: filer_pb.LookupVolumeRequest
-	(*Locations)(nil),                     // 27: filer_pb.Locations
-	(*Location)(nil),                      // 28: filer_pb.Location
-	(*LookupVolumeResponse)(nil),          // 29: filer_pb.LookupVolumeResponse
-	(*Collection)(nil),                    // 30: filer_pb.Collection
-	(*CollectionListRequest)(nil),         // 31: filer_pb.CollectionListRequest
-	(*CollectionListResponse)(nil),        // 32: filer_pb.CollectionListResponse
-	(*DeleteCollectionRequest)(nil),       // 33: filer_pb.DeleteCollectionRequest
-	(*DeleteCollectionResponse)(nil),      // 34: filer_pb.DeleteCollectionResponse
-	(*StatisticsRequest)(nil),             // 35: filer_pb.StatisticsRequest
-	(*StatisticsResponse)(nil),            // 36: filer_pb.StatisticsResponse
-	(*GetFilerConfigurationRequest)(nil),  // 37: filer_pb.GetFilerConfigurationRequest
-	(*GetFilerConfigurationResponse)(nil), // 38: filer_pb.GetFilerConfigurationResponse
-	(*SubscribeMetadataRequest)(nil),      // 39: filer_pb.SubscribeMetadataRequest
-	(*SubscribeMetadataResponse)(nil),     // 40: filer_pb.SubscribeMetadataResponse
-	(*LogEntry)(nil),                      // 41: filer_pb.LogEntry
-	(*KeepConnectedRequest)(nil),          // 42: filer_pb.KeepConnectedRequest
-	(*KeepConnectedResponse)(nil),         // 43: filer_pb.KeepConnectedResponse
-	(*LocateBrokerRequest)(nil),           // 44: filer_pb.LocateBrokerRequest
-	(*LocateBrokerResponse)(nil),          // 45: filer_pb.LocateBrokerResponse
-	(*KvGetRequest)(nil),                  // 46: filer_pb.KvGetRequest
-	(*KvGetResponse)(nil),                 // 47: filer_pb.KvGetResponse
-	(*KvPutRequest)(nil),                  // 48: filer_pb.KvPutRequest
-	(*KvPutResponse)(nil),                 // 49: filer_pb.KvPutResponse
-	(*FilerConf)(nil),                     // 50: filer_pb.FilerConf
-	(*DownloadToLocalRequest)(nil),        // 51: filer_pb.DownloadToLocalRequest
-	(*DownloadToLocalResponse)(nil),       // 52: filer_pb.DownloadToLocalResponse
+	(*LookupDirectoryEntryRequest)(nil),             // 0: filer_pb.LookupDirectoryEntryRequest
+	(*LookupDirectoryEntryResponse)(nil),            // 1: filer_pb.LookupDirectoryEntryResponse
+	(*ListEntriesRequest)(nil),                      // 2: filer_pb.ListEntriesRequest
+	(*ListEntriesResponse)(nil),                     // 3: filer_pb.ListEntriesResponse
+	(*RemoteEntry)(nil),                             // 4: filer_pb.RemoteEntry
+	(*Entry)(nil),                                   // 5: filer_pb.Entry
+	(*FullEntry)(nil),                               // 6: filer_pb.FullEntry
+	(*EventNotification)(nil),                       // 7: filer_pb.EventNotification
+	(*FileChunk)(nil),                               // 8: filer_pb.FileChunk
+	(*FileChunkManifest)(nil),                       // 9: filer_pb.FileChunkManifest
+	(*FileId)(nil),                                  // 10: filer_pb.FileId
+	(*FuseAttributes)(nil),                          // 11: filer_pb.FuseAttributes
+	(*CreateEntryRequest)(nil),                      // 12: filer_pb.CreateEntryRequest
+	(*CreateEntryResponse)(nil),                     // 13: filer_pb.CreateEntryResponse
+	(*UpdateEntryRequest)(nil),                      // 14: filer_pb.UpdateEntryRequest
+	(*UpdateEntryResponse)(nil),                     // 15: filer_pb.UpdateEntryResponse
+	(*AppendToEntryRequest)(nil),                    // 16: filer_pb.AppendToEntryRequest
+	(*AppendToEntryResponse)(nil),                   // 17: filer_pb.AppendToEntryResponse
+	(*DeleteEntryRequest)(nil),                      // 18: filer_pb.DeleteEntryRequest
+	(*DeleteEntryResponse)(nil),                     // 19: filer_pb.DeleteEntryResponse
+	(*AtomicRenameEntryRequest)(nil),                // 20: filer_pb.AtomicRenameEntryRequest
+	(*AtomicRenameEntryResponse)(nil),               // 21: filer_pb.AtomicRenameEntryResponse
+	(*StreamRenameEntryRequest)(nil),                // 22: filer_pb.StreamRenameEntryRequest
+	(*StreamRenameEntryResponse)(nil),               // 23: filer_pb.StreamRenameEntryResponse
+	(*AssignVolumeRequest)(nil),                     // 24: filer_pb.AssignVolumeRequest
+	(*AssignVolumeResponse)(nil),                    // 25: filer_pb.AssignVolumeResponse
+	(*LookupVolumeRequest)(nil),                     // 26: filer_pb.LookupVolumeRequest
+	(*Locations)(nil),                               // 27: filer_pb.Locations
+	(*Location)(nil),                                // 28: filer_pb.Location
+	(*LookupVolumeResponse)(nil),                    // 29: filer_pb.LookupVolumeResponse
+	(*Collection)(nil),                              // 30: filer_pb.Collection
+	(*CollectionListRequest)(nil),                   // 31: filer_pb.CollectionListRequest
+	(*CollectionListResponse)(nil),                  // 32: filer_pb.CollectionListResponse
+	(*DeleteCollectionRequest)(nil),                 // 33: filer_pb.DeleteCollectionRequest
+	(*DeleteCollectionResponse)(nil),                // 34: filer_pb.DeleteCollectionResponse
+	(*StatisticsRequest)(nil),                       // 35: filer_pb.StatisticsRequest
+	(*StatisticsResponse)(nil),                      // 36: filer_pb.StatisticsResponse
+	(*GetFilerConfigurationRequest)(nil),            // 37: filer_pb.GetFilerConfigurationRequest
+	(*GetFilerConfigurationResponse)(nil),           // 38: filer_pb.GetFilerConfigurationResponse
+	(*SubscribeMetadataRequest)(nil),                // 39: filer_pb.SubscribeMetadataRequest
+	(*SubscribeMetadataResponse)(nil),               // 40: filer_pb.SubscribeMetadataResponse
+	(*LogEntry)(nil),                                // 41: filer_pb.LogEntry
+	(*KeepConnectedRequest)(nil),                    // 42: filer_pb.KeepConnectedRequest
+	(*KeepConnectedResponse)(nil),                   // 43: filer_pb.KeepConnectedResponse
+	(*LocateBrokerRequest)(nil),                     // 44: filer_pb.LocateBrokerRequest
+	(*LocateBrokerResponse)(nil),                    // 45: filer_pb.LocateBrokerResponse
+	(*KvGetRequest)(nil),                            // 46: filer_pb.KvGetRequest
+	(*KvGetResponse)(nil),                           // 47: filer_pb.KvGetResponse
+	(*KvPutRequest)(nil),                            // 48: filer_pb.KvPutRequest
+	(*KvPutResponse)(nil),                           // 49: filer_pb.KvPutResponse
+	(*FilerConf)(nil),                               // 50: filer_pb.FilerConf
+	(*CacheRemoteObjectToLocalClusterRequest)(nil),  // 51: filer_pb.CacheRemoteObjectToLocalClusterRequest
+	(*CacheRemoteObjectToLocalClusterResponse)(nil), // 52: filer_pb.CacheRemoteObjectToLocalClusterResponse
 	nil,                                   // 53: filer_pb.Entry.ExtendedEntry
 	nil,                                   // 54: filer_pb.LookupVolumeResponse.LocationsMapEntry
 	(*LocateBrokerResponse_Resource)(nil), // 55: filer_pb.LocateBrokerResponse.Resource
@@ -4269,7 +4274,7 @@ var file_filer_proto_depIdxs = []int32{
 	7,  // 20: filer_pb.SubscribeMetadataResponse.event_notification:type_name -> filer_pb.EventNotification
 	55, // 21: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource
 	56, // 22: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf
-	5,  // 23: filer_pb.DownloadToLocalResponse.entry:type_name -> filer_pb.Entry
+	5,  // 23: filer_pb.CacheRemoteObjectToLocalClusterResponse.entry:type_name -> filer_pb.Entry
 	27, // 24: filer_pb.LookupVolumeResponse.LocationsMapEntry.value:type_name -> filer_pb.Locations
 	0,  // 25: filer_pb.SeaweedFiler.LookupDirectoryEntry:input_type -> filer_pb.LookupDirectoryEntryRequest
 	2,  // 26: filer_pb.SeaweedFiler.ListEntries:input_type -> filer_pb.ListEntriesRequest
@@ -4291,7 +4296,7 @@ var file_filer_proto_depIdxs = []int32{
 	44, // 42: filer_pb.SeaweedFiler.LocateBroker:input_type -> filer_pb.LocateBrokerRequest
 	46, // 43: filer_pb.SeaweedFiler.KvGet:input_type -> filer_pb.KvGetRequest
 	48, // 44: filer_pb.SeaweedFiler.KvPut:input_type -> filer_pb.KvPutRequest
-	51, // 45: filer_pb.SeaweedFiler.DownloadToLocal:input_type -> filer_pb.DownloadToLocalRequest
+	51, // 45: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:input_type -> filer_pb.CacheRemoteObjectToLocalClusterRequest
 	1,  // 46: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse
 	3,  // 47: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse
 	13, // 48: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse
@@ -4312,7 +4317,7 @@ var file_filer_proto_depIdxs = []int32{
 	45, // 63: filer_pb.SeaweedFiler.LocateBroker:output_type -> filer_pb.LocateBrokerResponse
 	47, // 64: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse
 	49, // 65: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse
-	52, // 66: filer_pb.SeaweedFiler.DownloadToLocal:output_type -> filer_pb.DownloadToLocalResponse
+	52, // 66: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:output_type -> filer_pb.CacheRemoteObjectToLocalClusterResponse
 	46, // [46:67] is the sub-list for method output_type
 	25, // [25:46] is the sub-list for method input_type
 	25, // [25:25] is the sub-list for extension type_name
@@ -4939,7 +4944,7 @@ func file_filer_proto_init() {
 			}
 		}
 		file_filer_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*DownloadToLocalRequest); i {
+			switch v := v.(*CacheRemoteObjectToLocalClusterRequest); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -4951,7 +4956,7 @@ func file_filer_proto_init() {
 			}
 		}
 		file_filer_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*DownloadToLocalResponse); i {
+			switch v := v.(*CacheRemoteObjectToLocalClusterResponse); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -5039,7 +5044,7 @@ type SeaweedFilerClient interface {
 	LocateBroker(ctx context.Context, in *LocateBrokerRequest, opts ...grpc.CallOption) (*LocateBrokerResponse, error)
 	KvGet(ctx context.Context, in *KvGetRequest, opts ...grpc.CallOption) (*KvGetResponse, error)
 	KvPut(ctx context.Context, in *KvPutRequest, opts ...grpc.CallOption) (*KvPutResponse, error)
-	DownloadToLocal(ctx context.Context, in *DownloadToLocalRequest, opts ...grpc.CallOption) (*DownloadToLocalResponse, error)
+	CacheRemoteObjectToLocalCluster(ctx context.Context, in *CacheRemoteObjectToLocalClusterRequest, opts ...grpc.CallOption) (*CacheRemoteObjectToLocalClusterResponse, error)
 }

 type seaweedFilerClient struct {
@@ -5344,9 +5349,9 @@ func (c *seaweedFilerClient) KvPut(ctx context.Context, in *KvPutRequest, opts .
 	return out, nil
 }

-func (c *seaweedFilerClient) DownloadToLocal(ctx context.Context, in *DownloadToLocalRequest, opts ...grpc.CallOption) (*DownloadToLocalResponse, error) {
-	out := new(DownloadToLocalResponse)
-	err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/DownloadToLocal", in, out, opts...)
+func (c *seaweedFilerClient) CacheRemoteObjectToLocalCluster(ctx context.Context, in *CacheRemoteObjectToLocalClusterRequest, opts ...grpc.CallOption) (*CacheRemoteObjectToLocalClusterResponse, error) {
+	out := new(CacheRemoteObjectToLocalClusterResponse)
+	err := c.cc.Invoke(ctx, "/filer_pb.SeaweedFiler/CacheRemoteObjectToLocalCluster", in, out, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -5375,7 +5380,7 @@ type SeaweedFilerServer interface {
 	LocateBroker(context.Context, *LocateBrokerRequest) (*LocateBrokerResponse, error)
 	KvGet(context.Context, *KvGetRequest) (*KvGetResponse, error)
 	KvPut(context.Context, *KvPutRequest) (*KvPutResponse, error)
-	DownloadToLocal(context.Context, *DownloadToLocalRequest) (*DownloadToLocalResponse, error)
+	CacheRemoteObjectToLocalCluster(context.Context, *CacheRemoteObjectToLocalClusterRequest) (*CacheRemoteObjectToLocalClusterResponse, error)
 }

 // UnimplementedSeaweedFilerServer can be embedded to have forward compatible implementations.
@@ -5442,8 +5447,8 @@ func (*UnimplementedSeaweedFilerServer) KvGet(context.Context, *KvGetRequest) (*
 func (*UnimplementedSeaweedFilerServer) KvPut(context.Context, *KvPutRequest) (*KvPutResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method KvPut not implemented")
 }
-func (*UnimplementedSeaweedFilerServer) DownloadToLocal(context.Context, *DownloadToLocalRequest) (*DownloadToLocalResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method DownloadToLocal not implemented")
+func (*UnimplementedSeaweedFilerServer) CacheRemoteObjectToLocalCluster(context.Context, *CacheRemoteObjectToLocalClusterRequest) (*CacheRemoteObjectToLocalClusterResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method CacheRemoteObjectToLocalCluster not implemented")
 }

 func RegisterSeaweedFilerServer(s *grpc.Server, srv SeaweedFilerServer) {
@@ -5830,20 +5835,20 @@ func _SeaweedFiler_KvPut_Handler(srv interface{}, ctx context.Context, dec func(
 	return interceptor(ctx, in, info, handler)
 }

-func _SeaweedFiler_DownloadToLocal_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(DownloadToLocalRequest)
+func _SeaweedFiler_CacheRemoteObjectToLocalCluster_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CacheRemoteObjectToLocalClusterRequest)
 	if err := dec(in); err != nil {
 		return nil, err
 	}
 	if interceptor == nil {
-		return srv.(SeaweedFilerServer).DownloadToLocal(ctx, in)
+		return srv.(SeaweedFilerServer).CacheRemoteObjectToLocalCluster(ctx, in)
 	}
 	info := &grpc.UnaryServerInfo{
 		Server:     srv,
-		FullMethod: "/filer_pb.SeaweedFiler/DownloadToLocal",
+		FullMethod: "/filer_pb.SeaweedFiler/CacheRemoteObjectToLocalCluster",
 	}
 	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(SeaweedFilerServer).DownloadToLocal(ctx, req.(*DownloadToLocalRequest))
+		return srv.(SeaweedFilerServer).CacheRemoteObjectToLocalCluster(ctx, req.(*CacheRemoteObjectToLocalClusterRequest))
 	}
 	return interceptor(ctx, in, info, handler)
 }
@@ -5913,8 +5918,8 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 			Handler:    _SeaweedFiler_KvPut_Handler,
 		},
 		{
-			MethodName: "DownloadToLocal",
-			Handler:    _SeaweedFiler_DownloadToLocal_Handler,
+			MethodName: "CacheRemoteObjectToLocalCluster",
+			Handler:    _SeaweedFiler_CacheRemoteObjectToLocalCluster_Handler,
 		},
 	},
 	Streams: []grpc.StreamDesc{
@@ -7,209 +7,222 @@ option go_package = "github.com/chrislusf/seaweedfs/weed/pb/master_pb";
 //////////////////////////////////////////////////

 service Seaweed {
-    rpc SendHeartbeat (stream Heartbeat) returns (stream HeartbeatResponse) {
-    }
-    rpc KeepConnected (stream KeepConnectedRequest) returns (stream VolumeLocation) {
-    }
-    rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
-    }
-    rpc Assign (AssignRequest) returns (AssignResponse) {
-    }
-    rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
-    }
-    rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
-    }
-    rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) {
-    }
-    rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) {
-    }
-    rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) {
-    }
-    rpc VacuumVolume (VacuumVolumeRequest) returns (VacuumVolumeResponse) {
-    }
-    rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) {
-    }
-    rpc ListMasterClients (ListMasterClientsRequest) returns (ListMasterClientsResponse) {
-    }
-    rpc LeaseAdminToken (LeaseAdminTokenRequest) returns (LeaseAdminTokenResponse) {
-    }
-    rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) {
-    }
+  rpc SendHeartbeat (stream Heartbeat) returns (stream HeartbeatResponse) {
+  }
+  rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) {
+  }
+  rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
+  }
+  rpc Assign (AssignRequest) returns (AssignResponse) {
+  }
+  rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
+  }
+  rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
+  }
+  rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) {
+  }
+  rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) {
+  }
+  rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) {
+  }
+  rpc VacuumVolume (VacuumVolumeRequest) returns (VacuumVolumeResponse) {
+  }
+  rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) {
+  }
+  rpc ListClusterNodes (ListClusterNodesRequest) returns (ListClusterNodesResponse) {
+  }
+  rpc LeaseAdminToken (LeaseAdminTokenRequest) returns (LeaseAdminTokenResponse) {
+  }
+  rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) {
+  }

 }

 //////////////////////////////////////////////////

 message Heartbeat {
-    string ip = 1;
-    uint32 port = 2;
-    string public_url = 3;
-    uint64 max_file_key = 5;
-    string data_center = 6;
-    string rack = 7;
-    uint32 admin_port = 8;
-    repeated VolumeInformationMessage volumes = 9;
-    // delta volumes
-    repeated VolumeShortInformationMessage new_volumes = 10;
-    repeated VolumeShortInformationMessage deleted_volumes = 11;
-    bool has_no_volumes = 12;
+  string ip = 1;
+  uint32 port = 2;
+  string public_url = 3;
+  uint64 max_file_key = 5;
+  string data_center = 6;
+  string rack = 7;
+  uint32 admin_port = 8;
+  repeated VolumeInformationMessage volumes = 9;
+  // delta volumes
+  repeated VolumeShortInformationMessage new_volumes = 10;
+  repeated VolumeShortInformationMessage deleted_volumes = 11;
+  bool has_no_volumes = 12;

-    // erasure coding
-    repeated VolumeEcShardInformationMessage ec_shards = 16;
-    // delta erasure coding shards
-    repeated VolumeEcShardInformationMessage new_ec_shards = 17;
-    repeated VolumeEcShardInformationMessage deleted_ec_shards = 18;
-    bool has_no_ec_shards = 19;
+  // erasure coding
+  repeated VolumeEcShardInformationMessage ec_shards = 16;
+  // delta erasure coding shards
+  repeated VolumeEcShardInformationMessage new_ec_shards = 17;
+  repeated VolumeEcShardInformationMessage deleted_ec_shards = 18;
+  bool has_no_ec_shards = 19;

-    map<string, uint32> max_volume_counts = 4;
-    uint32 grpc_port = 20;
+  map<string, uint32> max_volume_counts = 4;
+  uint32 grpc_port = 20;

 }

 message HeartbeatResponse {
-    uint64 volume_size_limit = 1;
-    string leader = 2;
-    string metrics_address = 3;
-    uint32 metrics_interval_seconds = 4;
-    repeated StorageBackend storage_backends = 5;
+  uint64 volume_size_limit = 1;
+  string leader = 2;
+  string metrics_address = 3;
+  uint32 metrics_interval_seconds = 4;
+  repeated StorageBackend storage_backends = 5;
 }

 message VolumeInformationMessage {
-    uint32 id = 1;
-    uint64 size = 2;
-    string collection = 3;
-    uint64 file_count = 4;
-    uint64 delete_count = 5;
-    uint64 deleted_byte_count = 6;
-    bool read_only = 7;
-    uint32 replica_placement = 8;
-    uint32 version = 9;
-    uint32 ttl = 10;
-    uint32 compact_revision = 11;
-    int64 modified_at_second = 12;
-    string remote_storage_name = 13;
-    string remote_storage_key = 14;
-    string disk_type = 15;
+  uint32 id = 1;
+  uint64 size = 2;
+  string collection = 3;
+  uint64 file_count = 4;
+  uint64 delete_count = 5;
+  uint64 deleted_byte_count = 6;
+  bool read_only = 7;
+  uint32 replica_placement = 8;
+  uint32 version = 9;
+  uint32 ttl = 10;
+  uint32 compact_revision = 11;
+  int64 modified_at_second = 12;
+  string remote_storage_name = 13;
+  string remote_storage_key = 14;
+  string disk_type = 15;
 }

 message VolumeShortInformationMessage {
-    uint32 id = 1;
-    string collection = 3;
-    uint32 replica_placement = 8;
-    uint32 version = 9;
-    uint32 ttl = 10;
-    string disk_type = 15;
+  uint32 id = 1;
+  string collection = 3;
+  uint32 replica_placement = 8;
+  uint32 version = 9;
+  uint32 ttl = 10;
+  string disk_type = 15;
 }

 message VolumeEcShardInformationMessage {
-    uint32 id = 1;
-    string collection = 2;
-    uint32 ec_index_bits = 3;
-    string disk_type = 4;
+  uint32 id = 1;
+  string collection = 2;
+  uint32 ec_index_bits = 3;
+  string disk_type = 4;
 }

 message StorageBackend {
-    string type = 1;
-    string id = 2;
-    map<string, string> properties = 3;
+  string type = 1;
+  string id = 2;
+  map<string, string> properties = 3;
 }

 message Empty {
 }

 message SuperBlockExtra {
-    message ErasureCoding {
-        uint32 data = 1;
-        uint32 parity = 2;
-        repeated uint32 volume_ids = 3;
-    }
-    ErasureCoding erasure_coding = 1;
+  message ErasureCoding {
+    uint32 data = 1;
+    uint32 parity = 2;
+    repeated uint32 volume_ids = 3;
+  }
+  ErasureCoding erasure_coding = 1;
 }

 message KeepConnectedRequest {
-    string name = 1;
-    string client_address = 3;
+  string client_type = 1;
+  string client_address = 3;
+  string version = 4;
 }

 message VolumeLocation {
-    string url = 1;
-    string public_url = 2;
-    repeated uint32 new_vids = 3;
-    repeated uint32 deleted_vids = 4;
-    string leader = 5; // optional when leader is not itself
-    string data_center = 6; // optional when DataCenter is in use
-    uint32 grpc_port = 7;
+  string url = 1;
+  string public_url = 2;
+  repeated uint32 new_vids = 3;
+  repeated uint32 deleted_vids = 4;
+  string leader = 5; // optional when leader is not itself
+  string data_center = 6; // optional when DataCenter is in use
+  uint32 grpc_port = 7;
+}
+
+message ClusterNodeUpdate {
+  string node_type = 1;
+  string address = 2;
+  bool is_leader = 3;
+  bool is_add = 4;
+}
+
+message KeepConnectedResponse {
+  VolumeLocation volume_location = 1;
+  ClusterNodeUpdate cluster_node_update = 2;
 }

 message LookupVolumeRequest {
-    repeated string volume_or_file_ids = 1;
-    string collection = 2; // optional, a bit faster if provided.
+  repeated string volume_or_file_ids = 1;
+  string collection = 2; // optional, a bit faster if provided.
 }
 message LookupVolumeResponse {
-    message VolumeIdLocation {
-        string volume_or_file_id = 1;
-        repeated Location locations = 2;
-        string error = 3;
-        string auth = 4;
-    }
-    repeated VolumeIdLocation volume_id_locations = 1;
+  message VolumeIdLocation {
+    string volume_or_file_id = 1;
+    repeated Location locations = 2;
+    string error = 3;
+    string auth = 4;
+  }
+  repeated VolumeIdLocation volume_id_locations = 1;
 }

 message Location {
-    string url = 1;
-    string public_url = 2;
-    uint32 grpc_port = 3;
+  string url = 1;
+  string public_url = 2;
+  uint32 grpc_port = 3;
 }

 message AssignRequest {
-    uint64 count = 1;
-    string replication = 2;
-    string collection = 3;
-    string ttl = 4;
-    string data_center = 5;
-    string rack = 6;
-    string data_node = 7;
-    uint32 memory_map_max_size_mb = 8;
-    uint32 Writable_volume_count = 9;
-    string disk_type = 10;
+  uint64 count = 1;
+  string replication = 2;
+  string collection = 3;
+  string ttl = 4;
+  string data_center = 5;
+  string rack = 6;
+  string data_node = 7;
+  uint32 memory_map_max_size_mb = 8;
+  uint32 Writable_volume_count = 9;
+  string disk_type = 10;
 }
 message AssignResponse {
-    string fid = 1;
-    uint64 count = 4;
-    string error = 5;
-    string auth = 6;
-    repeated Location replicas = 7;
-    Location location = 8;
+  string fid = 1;
+  uint64 count = 4;
+  string error = 5;
+  string auth = 6;
+  repeated Location replicas = 7;
+  Location location = 8;
 }

 message StatisticsRequest {
-    string replication = 1;
-    string collection = 2;
-    string ttl = 3;
-    string disk_type = 4;
+  string replication = 1;
+  string collection = 2;
+  string ttl = 3;
+  string disk_type = 4;
 }
 message StatisticsResponse {
-    uint64 total_size = 4;
-    uint64 used_size = 5;
-    uint64 file_count = 6;
+  uint64 total_size = 4;
+  uint64 used_size = 5;
+  uint64 file_count = 6;
 }

 //
 // collection related
 //
 message Collection {
-    string name = 1;
+  string name = 1;
 }
 message CollectionListRequest {
-    bool include_normal_volumes = 1;
-    bool include_ec_volumes = 2;
+  bool include_normal_volumes = 1;
+  bool include_ec_volumes = 2;
 }
 message CollectionListResponse {
-    repeated Collection collections = 1;
+  repeated Collection collections = 1;
 }

 message CollectionDeleteRequest {
-    string name = 1;
+  string name = 1;
 }
 message CollectionDeleteResponse {
 }
@@ -218,56 +231,56 @@ message CollectionDeleteResponse {
 // volume related
 //
 message DiskInfo {
-    string type = 1;
-    int64 volume_count = 2;
-    int64 max_volume_count = 3;
-    int64 free_volume_count = 4;
-    int64 active_volume_count = 5;
-    repeated VolumeInformationMessage volume_infos = 6;
-    repeated VolumeEcShardInformationMessage ec_shard_infos = 7;
-    int64 remote_volume_count = 8;
+  string type = 1;
+  int64 volume_count = 2;
+  int64 max_volume_count = 3;
+  int64 free_volume_count = 4;
+  int64 active_volume_count = 5;
+  repeated VolumeInformationMessage volume_infos = 6;
+  repeated VolumeEcShardInformationMessage ec_shard_infos = 7;
+  int64 remote_volume_count = 8;
 }
 message DataNodeInfo {
-    string id = 1;
-    map<string, DiskInfo> diskInfos = 2;
-    uint32 grpc_port = 3;
+  string id = 1;
+  map<string, DiskInfo> diskInfos = 2;
+  uint32 grpc_port = 3;
 }
 message RackInfo {
-    string id = 1;
-    repeated DataNodeInfo data_node_infos = 2;
-    map<string, DiskInfo> diskInfos = 3;
+  string id = 1;
+  repeated DataNodeInfo data_node_infos = 2;
+  map<string, DiskInfo> diskInfos = 3;
 }
 message DataCenterInfo {
-    string id = 1;
-    repeated RackInfo rack_infos = 2;
-    map<string, DiskInfo> diskInfos = 3;
+  string id = 1;
+  repeated RackInfo rack_infos = 2;
+  map<string, DiskInfo> diskInfos = 3;
 }
 message TopologyInfo {
-    string id = 1;
-    repeated DataCenterInfo data_center_infos = 2;
-    map<string, DiskInfo> diskInfos = 3;
+  string id = 1;
+  repeated DataCenterInfo data_center_infos = 2;
+  map<string, DiskInfo> diskInfos = 3;
 }
 message VolumeListRequest {
 }
 message VolumeListResponse {
-    TopologyInfo topology_info = 1;
-    uint64 volume_size_limit_mb = 2;
+  TopologyInfo topology_info = 1;
+  uint64 volume_size_limit_mb = 2;
 }

 message LookupEcVolumeRequest {
-    uint32 volume_id = 1;
+  uint32 volume_id = 1;
 }
 message LookupEcVolumeResponse {
-    uint32 volume_id = 1;
-    message EcShardIdLocation {
-        uint32 shard_id = 1;
-        repeated Location locations = 2;
-    }
-    repeated EcShardIdLocation shard_id_locations = 2;
+  uint32 volume_id = 1;
+  message EcShardIdLocation {
+    uint32 shard_id = 1;
+    repeated Location locations = 2;
+  }
+  repeated EcShardIdLocation shard_id_locations = 2;
 }

 message VacuumVolumeRequest {
-    float garbage_threshold = 1;
+  float garbage_threshold = 1;
 }
 message VacuumVolumeResponse {
 }
@@ -275,37 +288,42 @@ message VacuumVolumeResponse {
 message GetMasterConfigurationRequest {
 }
 message GetMasterConfigurationResponse {
-    string metrics_address = 1;
-    uint32 metrics_interval_seconds = 2;
-    repeated StorageBackend storage_backends = 3;
-    string default_replication = 4;
-    string leader = 5;
-    uint32 volume_size_limit_m_b = 6;
-    bool volume_preallocate = 7;
+  string metrics_address = 1;
+  uint32 metrics_interval_seconds = 2;
+  repeated StorageBackend storage_backends = 3;
+  string default_replication = 4;
+  string leader = 5;
+  uint32 volume_size_limit_m_b = 6;
+  bool volume_preallocate = 7;
 }

-message ListMasterClientsRequest {
-    string client_type = 1;
+message ListClusterNodesRequest {
+  string client_type = 1;
 }
-message ListMasterClientsResponse {
-    repeated string grpc_addresses = 1;
+message ListClusterNodesResponse {
+  message ClusterNode {
+    string address = 1;
+    string version = 2;
+    bool is_leader = 3;
+  }
+  repeated ClusterNode cluster_nodes = 1;
 }

 message LeaseAdminTokenRequest {
-    int64 previous_token = 1;
-    int64 previous_lock_time = 2;
-    string lock_name = 3;
-    string client_name = 4;
+  int64 previous_token = 1;
+  int64 previous_lock_time = 2;
+  string lock_name = 3;
+  string client_name = 4;
 }
 message LeaseAdminTokenResponse {
-    int64 token = 1;
-    int64 lock_ts_ns = 2;
+  int64 token = 1;
+  int64 lock_ts_ns = 2;
 }

 message ReleaseAdminTokenRequest {
-    int64 previous_token = 1;
-    int64 previous_lock_time = 2;
-    string lock_name = 3;
+  int64 previous_token = 1;
+  int64 previous_lock_time = 2;
+  string lock_name = 3;
 }
 message ReleaseAdminTokenResponse {
 }
@@ -14,7 +14,7 @@ service VolumeServer {

    rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
    }
-    rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (VacuumVolumeCompactResponse) {
+    rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) {
    }
    rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
    }
@@ -47,7 +47,7 @@ service VolumeServer {
    }

    // copy the .idx .dat files, and mount this volume
-    rpc VolumeCopy (VolumeCopyRequest) returns (VolumeCopyResponse) {
+    rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
    }
    rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
    }
@@ -142,6 +142,7 @@ message VacuumVolumeCompactRequest {
    int64 preallocate = 2;
 }
 message VacuumVolumeCompactResponse {
+    int64 processed_bytes = 1;
 }

 message VacuumVolumeCommitRequest {
@@ -251,6 +252,7 @@ message VolumeCopyRequest {
 }
 message VolumeCopyResponse {
    uint64 last_append_at_ns = 1;
+    int64 processed_bytes = 2;
 }

 message CopyFileRequest {
@@ -83,6 +83,7 @@ func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation
 	}
 	isLastPage := false
 	for !isLastPage && err == nil {
+		var localErr error
 		listErr := s.conn.ListObjectsV2Pages(listInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
 			for _, content := range page.Contents {
 				key := *content.Key
@@ -94,6 +95,7 @@ func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation
 					RemoteETag:  *content.ETag,
 					StorageName: s.conf.Name,
 				}); err != nil {
+					localErr = err
 					return false
 				}
 			}
@@ -104,6 +106,9 @@ func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation
 		if listErr != nil {
 			err = fmt.Errorf("list %v: %v", remote, listErr)
 		}
+		if localErr != nil {
+			err = fmt.Errorf("process %v: %v", remote, localErr)
+		}
 	}
 	return
 }
@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"sync"

 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -23,6 +24,8 @@ type Iam interface {
 }

 type IdentityAccessManagement struct {
+	m sync.RWMutex
+
 	identities []*Identity
 	domain     string
 }
@@ -131,31 +134,38 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 		}
 		identities = append(identities, t)
 	}
-
+	iam.m.Lock()
 	// atomically switch
 	iam.identities = identities
+	iam.m.Unlock()
 	return nil
 }

 func (iam *IdentityAccessManagement) isEnabled() bool {
-
+	iam.m.RLock()
+	defer iam.m.RUnlock()
 	return len(iam.identities) > 0
 }

 func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) {

+	iam.m.RLock()
+	defer iam.m.RUnlock()
 	for _, ident := range iam.identities {
 		for _, cred := range ident.Credentials {
+			// println("checking", ident.Name, cred.AccessKey)
 			if cred.AccessKey == accessKey {
 				return ident, cred, true
 			}
 		}
 	}
+	glog.V(1).Infof("could not find accessKey %s", accessKey)
 	return nil, nil, false
 }

 func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, found bool) {
-
+	iam.m.RLock()
+	defer iam.m.RUnlock()
 	for _, ident := range iam.identities {
 		if ident.Name == "anonymous" {
 			return ident, true
@@ -177,12 +187,14 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt
 				r.Header.Set(xhttp.AmzIdentityId, identity.Name)
 				if identity.isAdmin() {
 					r.Header.Set(xhttp.AmzIsAdmin, "true")
+				} else if _, ok := r.Header[xhttp.AmzIsAdmin]; ok {
+					r.Header.Del(xhttp.AmzIsAdmin)
 				}
 			}
 			f(w, r)
 			return
 		}
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 	}
 }

@@ -68,7 +68,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa

 	uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId

-	entries, _, err := s3a.list(uploadDirectory, "", "", false, 0)
+	entries, _, err := s3a.list(uploadDirectory, "", "", false, maxPartsList)
 	if err != nil || len(entries) == 0 {
 		glog.Errorf("completeMultipartUpload %s %s error: %v, entries:%d", *input.Bucket, *input.UploadId, err, len(entries))
 		return nil, s3err.ErrNoSuchUpload
@@ -36,7 +36,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
 	if s3a.iam.isEnabled() {
 		identity, s3Err = s3a.iam.authUser(r)
 		if s3Err != s3err.ErrNone {
-			s3err.WriteErrorResponse(w, s3Err, r)
+			s3err.WriteErrorResponse(w, r, s3Err)
 			return
 		}
 	}
@@ -46,7 +46,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
 	entries, _, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32)

 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}

@@ -73,7 +73,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
 		Buckets: buckets,
 	}

-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 }

 func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
@@ -100,17 +100,24 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
 		}
 		return nil
 	}); err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 	if exist, err := s3a.exists(s3a.option.BucketsPath, bucket, true); err == nil && exist {
 		errCode = s3err.ErrBucketAlreadyExists
 	}
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}

+	if s3a.iam.isEnabled() {
+		if _, errCode = s3a.iam.authRequest(r, s3_constants.ACTION_ADMIN); errCode != s3err.ErrNone {
+			s3err.WriteErrorResponse(w, r, errCode)
+			return
+		}
+	}
+
 	fn := func(entry *filer_pb.Entry) {
 		if identityId := r.Header.Get(xhttp.AmzIdentityId); identityId != "" {
 			if entry.Extended == nil {
@@ -123,11 +130,11 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
 	// create the folder for bucket, but lazily create actual collection
 	if err := s3a.mkdir(s3a.option.BucketsPath, bucket, fn); err != nil {
 		glog.Errorf("PutBucketHandler mkdir: %v", err)
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}

-	writeSuccessResponseEmpty(w)
+	writeSuccessResponseEmpty(w, r)
 }

 func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
@@ -136,7 +143,7 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
 	glog.V(3).Infof("DeleteBucketHandler %s", bucket)

 	if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, err, r)
+		s3err.WriteErrorResponse(w, r, err)
 		return
 	}

@@ -158,11 +165,11 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
 	err = s3a.rm(s3a.option.BucketsPath, bucket, false, true)

 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}

-	s3err.WriteEmptyResponse(w, http.StatusNoContent)
+	s3err.WriteEmptyResponse(w, r, http.StatusNoContent)
 }

 func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
@@ -170,12 +177,12 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request
 	bucket, _ := getBucketAndObject(r)
 	glog.V(3).Infof("HeadBucketHandler %s", bucket)

-	if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, err, r)
+	if entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket); entry == nil || err == filer_pb.ErrNotFound {
+		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
 		return
 	}

-	writeSuccessResponseEmpty(w)
+	writeSuccessResponseEmpty(w, r)
 }

 func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorCode {
@@ -216,7 +223,7 @@ func (s3a *S3ApiServer) GetBucketAclHandler(w http.ResponseWriter, r *http.Reque
 	glog.V(3).Infof("GetBucketAclHandler %s", bucket)

 	if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, err, r)
+		s3err.WriteErrorResponse(w, r, err)
 		return
 	}

@@ -245,7 +252,7 @@ func (s3a *S3ApiServer) GetBucketAclHandler(w http.ResponseWriter, r *http.Reque
 			})
 		}
 	}
-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 }

 // GetBucketLifecycleConfigurationHandler Get Bucket Lifecycle configuration
@@ -256,18 +263,19 @@ func (s3a *S3ApiServer) GetBucketLifecycleConfigurationHandler(w http.ResponseWr
 	glog.V(3).Infof("GetBucketAclHandler %s", bucket)

 	if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, err, r)
+		s3err.WriteErrorResponse(w, r, err)
 		return
 	}
 	fc, err := filer.ReadFilerConf(s3a.option.Filer, s3a.option.GrpcDialOption, nil)
 	if err != nil {
 		glog.Errorf("GetBucketLifecycleConfigurationHandler: %s", err)
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 	ttls := fc.GetCollectionTtls(bucket)
 	if len(ttls) == 0 {
-		s3err.WriteErrorResponse(w, s3err.ErrNoSuchLifecycleConfiguration, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchLifecycleConfiguration)
+		return
 	}
 	response := Lifecycle{}
 	for prefix, internalTtl := range ttls {
@@ -284,14 +292,14 @@ func (s3a *S3ApiServer) GetBucketLifecycleConfigurationHandler(w http.ResponseWr
 			Expiration: Expiration{Days: days, set: true},
 		})
 	}
-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 }

 // PutBucketLifecycleConfigurationHandler Put Bucket Lifecycle configuration
 // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html
 func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) {

-	s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
+	s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)

 }

@@ -299,6 +307,6 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr
 // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketLifecycle.html
 func (s3a *S3ApiServer) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {

-	s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
+	s3err.WriteEmptyResponse(w, r, http.StatusNoContent)

 }
@@ -26,12 +26,12 @@ func (s3a *S3ApiServer) AdjustedUrl(location *filer_pb.Location) string {
 	return location.Url
 }

-func writeSuccessResponseXML(w http.ResponseWriter, response interface{}) {
-	s3err.WriteXMLResponse(w, http.StatusOK, response)
+func writeSuccessResponseXML(w http.ResponseWriter, r *http.Request, response interface{}) {
+	s3err.WriteXMLResponse(w, r, http.StatusOK, response)
 }

-func writeSuccessResponseEmpty(w http.ResponseWriter) {
-	s3err.WriteEmptyResponse(w, http.StatusOK)
+func writeSuccessResponseEmpty(w http.ResponseWriter, r *http.Request) {
+	s3err.WriteEmptyResponse(w, r, http.StatusOK)
 }

 func validateContentMd5(h http.Header) ([]byte, error) {
@@ -34,16 +34,16 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 		dir, name := fullPath.DirAndName()
 		entry, err := s3a.getEntry(dir, name)
 		if err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 			return
 		}
 		entry.Extended = weed_server.SaveAmzMetaData(r, entry.Extended, isReplace(r))
 		err = s3a.touch(dir, name, entry)
 		if err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 			return
 		}
-		writeSuccessResponseXML(w, CopyObjectResult{
+		writeSuccessResponseXML(w, r, CopyObjectResult{
 			ETag:         fmt.Sprintf("%x", entry.Attributes.Md5),
 			LastModified: time.Now().UTC(),
 		})
@@ -52,19 +52,19 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request

 	// If source object is empty or bucket is empty, reply back invalid copy source.
 	if srcObject == "" || srcBucket == "" {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 		return
 	}
 	srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
 	dir, name := srcPath.DirAndName()
 	_, err = s3a.getEntry(dir, name)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 		return
 	}

 	if srcBucket == dstBucket && srcObject == dstObject {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopyDest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopyDest)
 		return
 	}

@@ -75,7 +75,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request

 	_, _, resp, err := util.DownloadFile(srcUrl, "")
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 		return
 	}
 	defer util.CloseResponse(resp)
@@ -84,7 +84,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 	etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body)

 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}

@@ -95,7 +95,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 		LastModified: time.Now().UTC(),
 	}

-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)

 }

@@ -128,7 +128,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 	srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
 	// If source object is empty or bucket is empty, reply back invalid copy source.
 	if srcObject == "" || srcBucket == "" {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 		return
 	}

@@ -137,7 +137,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req

 	partID, err := strconv.Atoi(partIDString)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidPart, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPart)
 		return
 	}

@@ -145,7 +145,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req

 	// check partID with maximum part ID for multipart objects
 	if partID > globalMaxPartID {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts)
 		return
 	}

@@ -158,7 +158,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req

 	dataReader, err := util.ReadUrlAsReaderCloser(srcUrl, rangeHeader)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
 		return
 	}
 	defer dataReader.Close()
@@ -167,7 +167,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 	etag, errCode := s3a.putToFiler(r, dstUrl, dataReader)

 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}

@@ -178,7 +178,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 		LastModified: time.Now().UTC(),
 	}

-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)

 }

@@ -39,9 +39,12 @@ func init() {

 func mimeDetect(r *http.Request, dataReader io.Reader) io.ReadCloser {
 	mimeBuffer := make([]byte, 512)
-	dataReader.Read(mimeBuffer)
-	r.Header.Set("Content-Type", http.DetectContentType(mimeBuffer))
-	return io.NopCloser(io.MultiReader(bytes.NewReader(mimeBuffer), dataReader))
+	size, _ := dataReader.Read(mimeBuffer)
+	if size > 0 {
+		r.Header.Set("Content-Type", http.DetectContentType(mimeBuffer[:size]))
+		return io.NopCloser(io.MultiReader(bytes.NewReader(mimeBuffer[:size]), dataReader))
+	}
+	return io.NopCloser(dataReader)
 }

 func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
@@ -53,20 +56,20 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)

 	_, err := validateContentMd5(r.Header)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidDigest)
 		return
 	}

 	if r.Header.Get("Cache-Control") != "" {
 		if _, err = cacheobject.ParseRequestCacheControl(r.Header.Get("Cache-Control")); err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidDigest)
 			return
 		}
 	}

 	if r.Header.Get("Expires") != "" {
 		if _, err = time.Parse(http.TimeFormat, r.Header.Get("Expires")); err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidDigest)
 			return
 		}
 	}
@@ -84,12 +87,12 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
 			_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
 		}
 		if s3ErrCode != s3err.ErrNone {
-			s3err.WriteErrorResponse(w, s3ErrCode, r)
+			s3err.WriteErrorResponse(w, r, s3ErrCode)
 			return
 		}
 	} else {
 		if authTypeStreamingSigned == rAuthType {
-			s3err.WriteErrorResponse(w, s3err.ErrAuthNotSetup, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrAuthNotSetup)
 			return
 		}
 	}
@@ -97,7 +100,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)

 	if strings.HasSuffix(object, "/") {
 		if err := s3a.mkdir(s3a.option.BucketsPath, bucket+object, nil); err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 			return
 		}
 	} else {
@@ -110,14 +113,14 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
 		etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)

 		if errCode != s3err.ErrNone {
-			s3err.WriteErrorResponse(w, errCode, r)
+			s3err.WriteErrorResponse(w, r, errCode)
 			return
 		}

 		setEtag(w, etag)
 	}

-	writeSuccessResponseEmpty(w)
+	writeSuccessResponseEmpty(w, r)
 }

 func urlPathEscape(object string) string {
@@ -134,7 +137,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 	glog.V(3).Infof("GetObjectHandler %s %s", bucket, object)

 	if strings.HasSuffix(r.URL.Path, "/") {
-		s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
 		return
 	}

@@ -212,13 +215,13 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h

 	deleteXMLBytes, err := io.ReadAll(r.Body)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}

 	deleteObjects := &DeleteObjectsRequest{}
 	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrMalformedXML, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML)
 		return
 	}

@@ -270,7 +273,7 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 	}
 	deleteResp.Errors = deleteErrors

-	writeSuccessResponseXML(w, deleteResp)
+	writeSuccessResponseXML(w, r, deleteResp)

 }

@@ -314,7 +317,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des

 	if err != nil {
 		glog.Errorf("NewRequest %s: %v", destUrl, err)
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}

@@ -343,19 +346,19 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des

 	if postErr != nil {
 		glog.Errorf("post to filer: %v", postErr)
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 	defer util.CloseResponse(resp)

 	if resp.StatusCode == http.StatusPreconditionFailed {
-		s3err.WriteErrorResponse(w, s3err.ErrPreconditionFailed, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrPreconditionFailed)
 		return
 	}

 	if (resp.ContentLength == -1 || resp.StatusCode == 404) && resp.StatusCode != 304 {
 		if r.Method != "DELETE" {
-			s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
 			return
 		}
 	}
@@ -29,23 +29,23 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R

 	reader, err := r.MultipartReader()
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest)
 		return
 	}
 	form, err := reader.ReadForm(int64(5 * humanize.MiByte))
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest)
 		return
 	}
 	defer form.RemoveAll()

 	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(form)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest)
 		return
 	}
 	if fileBody == nil {
-		s3err.WriteErrorResponse(w, s3err.ErrPOSTFileRequired, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrPOSTFileRequired)
 		return
 	}
 	defer fileBody.Close()
@@ -63,7 +63,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 	if successRedirect != "" {
 		redirectURL, err = url.Parse(successRedirect)
 		if err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest)
 			return
 		}
 	}
@@ -71,13 +71,13 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 	// Verify policy signature.
 	errCode := s3a.iam.doesPolicySignatureMatch(formValues)
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}

 	policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedPOSTRequest)
 		return
 	}

@@ -86,7 +86,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R

 		postPolicyForm, err := policy.ParsePostPolicyForm(string(policyBytes))
 		if err != nil {
-			s3err.WriteErrorResponse(w, s3err.ErrPostPolicyConditionInvalidFormat, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrPostPolicyConditionInvalidFormat)
 			return
 		}

@@ -102,12 +102,12 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 		lengthRange := postPolicyForm.Conditions.ContentLengthRange
 		if lengthRange.Valid {
 			if fileSize < lengthRange.Min {
-				s3err.WriteErrorResponse(w, s3err.ErrEntityTooSmall, r)
+				s3err.WriteErrorResponse(w, r, s3err.ErrEntityTooSmall)
 				return
 			}

 			if fileSize > lengthRange.Max {
-				s3err.WriteErrorResponse(w, s3err.ErrEntityTooLarge, r)
+				s3err.WriteErrorResponse(w, r, s3err.ErrEntityTooLarge)
 				return
 			}
 		}
@@ -118,7 +118,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 	etag, errCode := s3a.putToFiler(r, uploadUrl, fileBody)

 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}

@@ -126,7 +126,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 		// Replace raw query params..
 		redirectURL.RawQuery = getRedirectPostRawQuery(bucket, object, etag)
 		w.Header().Set("Location", redirectURL.String())
-		s3err.WriteEmptyResponse(w, http.StatusSeeOther)
+		s3err.WriteEmptyResponse(w, r, http.StatusSeeOther)
 		return
 	}

@@ -141,11 +141,11 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 			ETag:     `"` + etag + `"`,
 			Location: w.Header().Get("Location"),
 		}
-		s3err.WriteXMLResponse(w, http.StatusCreated, resp)
+		s3err.WriteXMLResponse(w, r, http.StatusCreated, resp)
 	case "200":
-		s3err.WriteEmptyResponse(w, http.StatusOK)
+		s3err.WriteEmptyResponse(w, r, http.StatusOK)
 	default:
-		writeSuccessResponseEmpty(w)
+		writeSuccessResponseEmpty(w, r)
 	}

 }
@@ -45,11 +45,11 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http
 	glog.V(2).Info("NewMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode)

 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}

-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)

 }

@@ -69,11 +69,11 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
 	glog.V(2).Info("CompleteMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode)

 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}

-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)

 }

@@ -91,13 +91,13 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht
 	})

 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}

 	glog.V(2).Info("AbortMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)))

-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)

 }

@@ -107,13 +107,13 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht

 	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query())
 	if maxUploads < 0 {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxUploads, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxUploads)
 		return
 	}
 	if keyMarker != "" {
 		// Marker not common with prefix is not implemented.
 		if !strings.HasPrefix(keyMarker, prefix) {
-			s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
 			return
 		}
 	}
@@ -128,16 +128,16 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht
 		UploadIdMarker: aws.String(uploadIDMarker),
 	})

-	glog.V(2).Info("ListMultipartUploadsHandler", string(s3err.EncodeXMLResponse(response)), errCode)
+	glog.V(2).Infof("ListMultipartUploadsHandler %s errCode=%d", string(s3err.EncodeXMLResponse(response)), errCode)

 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}

 	// TODO handle encodingType

-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 }

 // ListObjectPartsHandler - Lists object parts in a multipart upload.
@@ -146,11 +146,11 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re

 	uploadID, partNumberMarker, maxParts, _ := getObjectResources(r.URL.Query())
 	if partNumberMarker < 0 {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidPartNumberMarker, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPartNumberMarker)
 		return
 	}
 	if maxParts < 0 {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts)
 		return
 	}

@@ -162,14 +162,14 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
 		UploadId:         aws.String(uploadID),
 	})

-	glog.V(2).Info("ListObjectPartsHandler", string(s3err.EncodeXMLResponse(response)), errCode)
+	glog.V(2).Infof("ListObjectPartsHandler %s count=%d", string(s3err.EncodeXMLResponse(response)), len(response.Part))

 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}

-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)

 }

@@ -180,18 +180,18 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
 	uploadID := r.URL.Query().Get("uploadId")
 	exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true)
 	if !exists {
-		s3err.WriteErrorResponse(w, s3err.ErrNoSuchUpload, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
 		return
 	}

 	partIDString := r.URL.Query().Get("partNumber")
 	partID, err := strconv.Atoi(partIDString)
 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidPart, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidPart)
 		return
 	}
 	if partID > globalMaxPartID {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxParts)
 		return
 	}

@@ -208,12 +208,14 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
 			_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
 		}
 		if s3ErrCode != s3err.ErrNone {
-			s3err.WriteErrorResponse(w, s3ErrCode, r)
+			s3err.WriteErrorResponse(w, r, s3ErrCode)
 			return
 		}
 	}
 	defer dataReader.Close()

+	glog.V(2).Infof("PutObjectPartHandler %s %s %04d", bucket, uploadID, partID)
+
 	uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part?collection=%s",
 		s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID, bucket)

@@ -223,13 +225,13 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ

 	etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)
 	if errCode != s3err.ErrNone {
-		s3err.WriteErrorResponse(w, errCode, r)
+		s3err.WriteErrorResponse(w, r, errCode)
 		return
 	}

 	setEtag(w, etag)

-	writeSuccessResponseEmpty(w)
+	writeSuccessResponseEmpty(w, r)

 }

@@ -26,15 +26,15 @@ func (s3a *S3ApiServer) GetObjectTaggingHandler(w http.ResponseWriter, r *http.R
 	if err != nil {
 		if err == filer_pb.ErrNotFound {
 			glog.Errorf("GetObjectTaggingHandler %s: %v", r.URL, err)
-			s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
 		} else {
 			glog.Errorf("GetObjectTaggingHandler %s: %v", r.URL, err)
-			s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		}
 		return
 	}

-	writeSuccessResponseXML(w, FromTags(tags))
+	writeSuccessResponseXML(w, r, FromTags(tags))

 }

@@ -52,29 +52,29 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R
 	input, err := io.ReadAll(io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		glog.Errorf("PutObjectTaggingHandler read input %s: %v", r.URL, err)
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}
 	if err = xml.Unmarshal(input, tagging); err != nil {
 		glog.Errorf("PutObjectTaggingHandler Unmarshal %s: %v", r.URL, err)
-		s3err.WriteErrorResponse(w, s3err.ErrMalformedXML, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML)
 		return
 	}
 	tags := tagging.ToTags()
 	if len(tags) > 10 {
 		glog.Errorf("PutObjectTaggingHandler tags %s: %d tags more than 10", r.URL, len(tags))
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag)
 		return
 	}
 	for k, v := range tags {
 		if len(k) > 128 {
 			glog.Errorf("PutObjectTaggingHandler tags %s: tag key %s longer than 128", r.URL, k)
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag)
 			return
 		}
 		if len(v) > 256 {
 			glog.Errorf("PutObjectTaggingHandler tags %s: tag value %s longer than 256", r.URL, v)
-			s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag)
 			return
 		}
 	}
@@ -82,15 +82,15 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R
 	if err = s3a.setTags(dir, name, tagging.ToTags()); err != nil {
 		if err == filer_pb.ErrNotFound {
 			glog.Errorf("PutObjectTaggingHandler setTags %s: %v", r.URL, err)
-			s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
 		} else {
 			glog.Errorf("PutObjectTaggingHandler setTags %s: %v", r.URL, err)
-			s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		}
 		return
 	}

-	w.WriteHeader(http.StatusNoContent)
+	w.WriteHeader(http.StatusOK)

 }

@@ -108,10 +108,10 @@ func (s3a *S3ApiServer) DeleteObjectTaggingHandler(w http.ResponseWriter, r *htt
 	if err != nil {
 		if err == filer_pb.ErrNotFound {
 			glog.Errorf("DeleteObjectTaggingHandler %s: %v", r.URL, err)
-			s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
 		} else {
 			glog.Errorf("DeleteObjectTaggingHandler %s: %v", r.URL, err)
-			s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		}
 		return
 	}
@@ -45,11 +45,11 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 	originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())

 	if maxKeys < 0 {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxKeys, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
 		return
 	}
 	if delimiter != "" && delimiter != "/" {
-		s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
 		return
 	}

@@ -61,13 +61,13 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)

 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}

 	if len(response.Contents) == 0 {
 		if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists {
-			s3err.WriteErrorResponse(w, s3err.ErrNoSuchBucket, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
 			return
 		}
 	}
@@ -87,7 +87,7 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 		StartAfter:            startAfter,
 	}

-	writeSuccessResponseXML(w, responseV2)
+	writeSuccessResponseXML(w, r, responseV2)
 }

 func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
@@ -101,29 +101,29 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 	originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())

 	if maxKeys < 0 {
-		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxKeys, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
 		return
 	}
 	if delimiter != "" && delimiter != "/" {
-		s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
 		return
 	}

 	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)

 	if err != nil {
-		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}

 	if len(response.Contents) == 0 {
 		if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists {
-			s3err.WriteErrorResponse(w, s3err.ErrNoSuchBucket, r)
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
 			return
 		}
 	}

-	writeSuccessResponseXML(w, response)
+	writeSuccessResponseXML(w, r, response)
 }

 func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, marker string, delimiter string) (response ListBucketResult, err error) {
@@ -220,12 +220,16 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 			err = subErr
 			return
 		}
+		counter += subCounter
 		isTruncated = isTruncated || subIsTruncated
 		maxKeys -= subCounter
 		nextMarker = subDir + "/" + subNextMarker
 		// finished processing this sub directory
 		marker = subDir
 	}
+	if maxKeys <= 0 {
+		return
+	}

 	// now marker is also a direct child of dir
 	request := &filer_pb.ListEntriesRequest{
@@ -66,7 +66,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 		// HeadObject
 		bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.HeadObjectHandler, ACTION_READ), "GET"))
 		// HeadBucket
-		bucket.Methods("HEAD").HandlerFunc(track(s3a.iam.Auth(s3a.HeadBucketHandler, ACTION_ADMIN), "GET"))
+		bucket.Methods("HEAD").HandlerFunc(track(s3a.iam.Auth(s3a.HeadBucketHandler, ACTION_READ), "GET"))

 		// CopyObjectPart
 		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", `.*?(\/|%2F).*?`).HandlerFunc(track(s3a.iam.Auth(s3a.CopyObjectPartHandler, ACTION_WRITE), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
@@ -103,13 +103,9 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(track(s3a.iam.Auth(s3a.CopyObjectHandler, ACTION_WRITE), "COPY"))
 		// PutObject
 		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.PutObjectHandler, ACTION_WRITE), "PUT"))
-		// PutBucket
-		bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketHandler, ACTION_ADMIN), "PUT"))

 		// DeleteObject
 		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteObjectHandler, ACTION_WRITE), "DELETE"))
-		// DeleteBucket
-		bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketHandler, ACTION_WRITE), "DELETE"))

 		// ListObjectsV2
 		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV2Handler, ACTION_LIST), "LIST")).Queries("list-type", "2")
@@ -139,6 +135,13 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {

 		// ListObjectsV1 (Legacy)
 		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV1Handler, ACTION_LIST), "LIST"))
+
+		// PutBucket
+		bucket.Methods("PUT").HandlerFunc(track(s3a.PutBucketHandler, "PUT"))
+
+		// DeleteBucket
+		bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketHandler, ACTION_WRITE), "DELETE"))
+
 		/*

 			// not implemented
@@ -4,5 +4,5 @@ import "net/http"

 func (s3a *S3ApiServer) StatusHandler(w http.ResponseWriter, r *http.Request) {
 	// write out the response code and content type header
-	writeSuccessResponseEmpty(w)
+	writeSuccessResponseEmpty(w, r)
 }
@@ -19,15 +19,15 @@ const (
 	MimeXML  mimeType = "application/xml"
 )

-func WriteXMLResponse(w http.ResponseWriter, statusCode int, response interface{}) {
-	WriteResponse(w, statusCode, EncodeXMLResponse(response), MimeXML)
+func WriteXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, response interface{}) {
+	WriteResponse(w, r, statusCode, EncodeXMLResponse(response), MimeXML)
 }

-func WriteEmptyResponse(w http.ResponseWriter, statusCode int) {
-	WriteResponse(w, statusCode, []byte{}, mimeNone)
+func WriteEmptyResponse(w http.ResponseWriter, r *http.Request, statusCode int) {
+	WriteResponse(w, r, statusCode, []byte{}, mimeNone)
 }

-func WriteErrorResponse(w http.ResponseWriter, errorCode ErrorCode, r *http.Request) {
+func WriteErrorResponse(w http.ResponseWriter, r *http.Request, errorCode ErrorCode) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -38,7 +38,7 @@ func WriteErrorResponse(w http.ResponseWriter, errorCode ErrorCode, r *http.Requ
 	apiError := GetAPIError(errorCode)
 	errorResponse := getRESTErrorResponse(apiError, r.URL.Path, bucket, object)
 	encodedErrorResponse := EncodeXMLResponse(errorResponse)
-	WriteResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, MimeXML)
+	WriteResponse(w, r, apiError.HTTPStatusCode, encodedErrorResponse, MimeXML)
 }

 func getRESTErrorResponse(err APIError, resource string, bucket, object string) RESTErrorResponse {
@@ -61,13 +61,17 @@ func EncodeXMLResponse(response interface{}) []byte {
 	return bytesBuffer.Bytes()
 }

-func setCommonHeaders(w http.ResponseWriter) {
+func setCommonHeaders(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("x-amz-request-id", fmt.Sprintf("%d", time.Now().UnixNano()))
 	w.Header().Set("Accept-Ranges", "bytes")
+	if r.Header.Get("Origin") != "" {
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		w.Header().Set("Access-Control-Allow-Credentials", "true")
+	}
 }

-func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
-	setCommonHeaders(w)
+func WriteResponse(w http.ResponseWriter, r *http.Request, statusCode int, response []byte, mType mimeType) {
+	setCommonHeaders(w, r)
 	if response != nil {
 		w.Header().Set("Content-Length", strconv.Itoa(len(response)))
 	}
@@ -88,5 +92,5 @@ func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType
 // If none of the http routes match respond with MethodNotAllowed
 func NotFoundHandler(w http.ResponseWriter, r *http.Request) {
 	glog.V(0).Infof("unsupported %s %s", r.Method, r.RequestURI)
-	WriteErrorResponse(w, ErrMethodNotAllowed, r)
+	WriteErrorResponse(w, r, ErrMethodNotAllowed)
 }
@@ -202,7 +202,7 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidTag: {
-		Code:           "InvalidArgument",
+		Code:           "InvalidTag",
 		Description:    "The Tag value you have provided is invalid",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
@@ -2,7 +2,6 @@ package s3api

 import (
 	stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
-	"github.com/chrislusf/seaweedfs/weed/util"
 	"net/http"
 	"strconv"
 	"time"
@@ -28,7 +27,7 @@ func (r *StatusRecorder) Flush() {

 func track(f http.HandlerFunc, action string) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Server", "SeaweedFS S3 "+util.VERSION)
+		w.Header().Set("Server", "SeaweedFS S3")
 		recorder := NewStatusResponseWriter(w)
 		start := time.Now()
 		f(recorder, r)
@@ -14,8 +14,9 @@ type TagSet struct {
 }

 type Tagging struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"`
+	XMLName xml.Name `xml:"Tagging"`
 	TagSet  TagSet   `xml:"TagSet"`
+	Xmlns   string   `xml:"xmlns,attr"`
 }

 func (t *Tagging) ToTags() map[string]string {
@@ -27,7 +28,7 @@ func (t *Tagging) ToTags() map[string]string {
 }

 func FromTags(tags map[string]string) (t *Tagging) {
-	t = &Tagging{}
+	t = &Tagging{Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/"}
 	for k, v := range tags {
 		t.TagSet.Tag = append(t.TagSet.Tag, Tag{
 			Key:   k,
@@ -32,6 +32,7 @@ func TestXMLUnmarshall(t *testing.T) {

 func TestXMLMarshall(t *testing.T) {
 	tags := &Tagging{
+		Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
 		TagSet: TagSet{
 			[]Tag{
 				{
@@ -3,6 +3,10 @@ package weed_server
 import (
 	"context"
 	"fmt"
+	"strings"
+	"sync"
+	"time"
+
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb"
@@ -12,12 +16,9 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/golang/protobuf/proto"
-	"strings"
-	"sync"
-	"time"
 )

-func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.DownloadToLocalRequest) (*filer_pb.DownloadToLocalResponse, error) {
+func (fs *FilerServer) CacheRemoteObjectToLocalCluster(ctx context.Context, req *filer_pb.CacheRemoteObjectToLocalClusterRequest) (*filer_pb.CacheRemoteObjectToLocalClusterResponse, error) {

 	// load all mappings
 	mappingEntry, err := fs.filer.FindEntry(ctx, util.JoinPath(filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE))
@@ -57,7 +58,7 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo
 		return nil, err
 	}

-	resp := &filer_pb.DownloadToLocalResponse{}
+	resp := &filer_pb.CacheRemoteObjectToLocalClusterResponse{}
 	if entry.Remote == nil || entry.Remote.RemoteSize == 0 {
 		return resp, nil
 	}
@@ -178,6 +179,7 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo
 	// this skips meta data log events

 	if err := fs.filer.Store.UpdateEntry(context.Background(), newEntry); err != nil {
+		fs.filer.DeleteChunks(chunks)
 		return nil, err
 	}
 	fs.filer.DeleteChunks(garbage)
@@ -161,7 +161,7 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, stream filer_pb.Seawee
 		if err := stream.Send(&filer_pb.StreamRenameEntryResponse{
 			Directory: string(newParent),
 			EventNotification: &filer_pb.EventNotification{
-				OldEntry:           &filer_pb.Entry{
+				OldEntry: &filer_pb.Entry{
 					Name: entry.Name(),
 				},
 				NewEntry:           newEntry.ToProtoEntry(),
@@ -61,7 +61,6 @@ type FilerOption struct {
 	recursiveDelete       bool
 	Cipher                bool
 	SaveToFilerLimit      int64
-	Filers                []pb.ServerAddress
 	ConcurrentUploadLimit int64
 }

@@ -108,7 +107,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 	fs.checkWithMaster()

 	go stats.LoopPushingMetric("filer", string(fs.option.Host), fs.metricsAddress, fs.metricsIntervalSec)
-	go fs.filer.KeepConnectedToMaster()
+	go fs.filer.KeepMasterClientConnected()

 	v := util.GetViper()
 	if !util.LoadConfiguration("filer", false) {
@@ -143,7 +142,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 		readonlyMux.HandleFunc("/", fs.readonlyFilerHandler)
 	}

-	fs.filer.AggregateFromPeers(option.Host, option.Filers)
+	fs.filer.AggregateFromPeers(option.Host)

 	fs.filer.LoadBuckets()

@@ -113,17 +113,15 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 	w.Header().Set("Access-Control-Expose-Headers", strings.Join(seaweedHeaders, ","))

 	//set tag count
-	if r.Method == "GET" {
-		tagCount := 0
-		for k := range entry.Extended {
-			if strings.HasPrefix(k, xhttp.AmzObjectTagging+"-") {
-				tagCount++
-			}
-		}
-		if tagCount > 0 {
-			w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount))
+	tagCount := 0
+	for k := range entry.Extended {
+		if strings.HasPrefix(k, xhttp.AmzObjectTagging+"-") {
+			tagCount++
 		}
 	}
+	if tagCount > 0 {
+		w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount))
+	}

 	if inm := r.Header.Get("If-None-Match"); inm == "\""+etag+"\"" {
 		w.WriteHeader(http.StatusNotModified)
@@ -171,11 +169,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 		chunks := entry.Chunks
 		if entry.IsInRemoteOnly() {
 			dir, name := entry.FullPath.DirAndName()
-			if resp, err := fs.DownloadToLocal(context.Background(), &filer_pb.DownloadToLocalRequest{
+			if resp, err := fs.CacheRemoteObjectToLocalCluster(context.Background(), &filer_pb.CacheRemoteObjectToLocalClusterRequest{
 				Directory: dir,
 				Name:      name,
 			}); err != nil {
-				glog.Errorf("DownloadToLocal %s: %v", entry.FullPath, err)
+				glog.Errorf("CacheRemoteObjectToLocalCluster %s: %v", entry.FullPath, err)
 				return fmt.Errorf("cache %s: %v", entry.FullPath, err)
 			} else {
 				chunks = resp.Entry.Chunks
@@ -97,6 +97,9 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite

 	md5bytes = md5Hash.Sum(nil)
 	filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent)
+	if replyerr != nil {
+		fs.filer.DeleteChunks(fileChunks)
+	}

 	return
 }
@@ -116,6 +119,9 @@ func (fs *FilerServer) doPutAutoChunk(ctx context.Context, w http.ResponseWriter

 	md5bytes = md5Hash.Sum(nil)
 	filerResult, replyerr = fs.saveMetaData(ctx, r, fileName, contentType, so, md5bytes, fileChunks, chunkOffset, smallContent)
+	if replyerr != nil {
+		fs.filer.DeleteChunks(fileChunks)
+	}

 	return
 }
@@ -218,13 +224,12 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
 	entry.Extended = SaveAmzMetaData(r, entry.Extended, false)

 	for k, v := range r.Header {
-		if len(v) > 0 && (strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition") {
+		if len(v) > 0 && len(v[0]) > 0 && (strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition") {
 			entry.Extended[k] = []byte(v[0])
 		}
 	}

 	if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil); dbErr != nil {
-		fs.filer.DeleteChunks(fileChunks)
 		replyerr = dbErr
 		filerResult.Error = dbErr.Error()
 		glog.V(0).Infof("failing to write %s to filer server : %v", path, dbErr)
@@ -327,6 +332,8 @@ func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool
 			tag := strings.Split(v, "=")
 			if len(tag) == 2 {
 				metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = []byte(tag[1])
+			} else if len(tag) == 1 {
+				metadata[xhttp.AmzObjectTagging+"-"+tag[0]] = nil
 			}
 		}
 	}
@@ -6,7 +6,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"net"
-	"strings"
 	"time"

 	"github.com/chrislusf/raft"
@@ -45,11 +44,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 			}

 			if len(message.DeletedVids) > 0 {
-				ms.clientChansLock.RLock()
-				for _, ch := range ms.clientChans {
-					ch <- message
-				}
-				ms.clientChansLock.RUnlock()
+				ms.broadcastToClients(&master_pb.KeepConnectedResponse{VolumeLocation: message})
 			}
 		}
 	}()
@@ -154,12 +149,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ

 		}
 		if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 {
-			ms.clientChansLock.RLock()
-			for host, ch := range ms.clientChans {
-				glog.V(0).Infof("master send to %s: %s", host, message.String())
-				ch <- message
-			}
-			ms.clientChansLock.RUnlock()
+			ms.broadcastToClients(&master_pb.KeepConnectedResponse{VolumeLocation: message})
 		}

 		// tell the volume servers about the leader
@@ -195,12 +185,20 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
 	// buffer by 1 so we don't end up getting stuck writing to stopChan forever
 	stopChan := make(chan bool, 1)

-	clientName, messageChan := ms.addClient(req.Name, peerAddress)
+	clientName, messageChan := ms.addClient(req.ClientType, peerAddress)
+	for _, update := range ms.Cluster.AddClusterNode(req.ClientType, peerAddress, req.Version) {
+		ms.broadcastToClients(update)
+	}

-	defer ms.deleteClient(clientName)
+	defer func() {
+		for _, update := range ms.Cluster.RemoveClusterNode(req.ClientType, peerAddress) {
+			ms.broadcastToClients(update)
+		}
+		ms.deleteClient(clientName)
+	}()

 	for _, message := range ms.Topo.ToVolumeLocations() {
-		if sendErr := stream.Send(message); sendErr != nil {
+		if sendErr := stream.Send(&master_pb.KeepConnectedResponse{VolumeLocation: message}); sendErr != nil {
 			return sendErr
 		}
 	}
@@ -235,21 +233,31 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ

 }

+func (ms *MasterServer) broadcastToClients(message *master_pb.KeepConnectedResponse) {
+	ms.clientChansLock.RLock()
+	for _, ch := range ms.clientChans {
+		ch <- message
+	}
+	ms.clientChansLock.RUnlock()
+}
+
 func (ms *MasterServer) informNewLeader(stream master_pb.Seaweed_KeepConnectedServer) error {
 	leader, err := ms.Topo.Leader()
 	if err != nil {
 		glog.Errorf("topo leader: %v", err)
 		return raft.NotLeaderError
 	}
-	if err := stream.Send(&master_pb.VolumeLocation{
-		Leader: string(leader),
+	if err := stream.Send(&master_pb.KeepConnectedResponse{
+		VolumeLocation: &master_pb.VolumeLocation{
+			Leader: string(leader),
+		},
 	}); err != nil {
 		return err
 	}
 	return nil
 }

-func (ms *MasterServer) addClient(clientType string, clientAddress pb.ServerAddress) (clientName string, messageChan chan *master_pb.VolumeLocation) {
+func (ms *MasterServer) addClient(clientType string, clientAddress pb.ServerAddress) (clientName string, messageChan chan *master_pb.KeepConnectedResponse) {
 	clientName = clientType + "@" + string(clientAddress)
 	glog.V(0).Infof("+ client %v", clientName)

@@ -258,7 +266,7 @@ func (ms *MasterServer) addClient(clientType string, clientAddress pb.ServerAddr
 	// trying to send to it in SendHeartbeat and so we can't lock the
 	// clientChansLock to remove the channel and we're stuck writing to it
 	// 100 is probably overkill
-	messageChan = make(chan *master_pb.VolumeLocation, 100)
+	messageChan = make(chan *master_pb.KeepConnectedResponse, 100)

 	ms.clientChansLock.Lock()
 	ms.clientChans[clientName] = messageChan
@@ -295,19 +303,6 @@ func findClientAddress(ctx context.Context, grpcPort uint32) string {

 }

-func (ms *MasterServer) ListMasterClients(ctx context.Context, req *master_pb.ListMasterClientsRequest) (*master_pb.ListMasterClientsResponse, error) {
-	resp := &master_pb.ListMasterClientsResponse{}
-	ms.clientChansLock.RLock()
-	defer ms.clientChansLock.RUnlock()
-
-	for k := range ms.clientChans {
-		if strings.HasPrefix(k, req.ClientType+"@") {
-			resp.GrpcAddresses = append(resp.GrpcAddresses, k[len(req.ClientType)+1:])
-		}
-	}
-	return resp, nil
-}
-
 func (ms *MasterServer) GetMasterConfiguration(ctx context.Context, req *master_pb.GetMasterConfigurationRequest) (*master_pb.GetMasterConfigurationResponse, error) {

 	// tell the volume servers about the leader
@@ -0,0 +1,21 @@
+package weed_server
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+)
+
+func (ms *MasterServer) ListClusterNodes(ctx context.Context, req *master_pb.ListClusterNodesRequest) (*master_pb.ListClusterNodesResponse, error) {
+	resp := &master_pb.ListClusterNodesResponse{}
+
+	clusterNodes := ms.Cluster.ListClusterNode(req.ClientType)
+
+	for _, node := range clusterNodes {
+		resp.ClusterNodes = append(resp.ClusterNodes, &master_pb.ListClusterNodesResponse_ClusterNode{
+			Address: string(node.Address),
+			Version: node.Version,
+			IsLeader: ms.Cluster.IsOneLeader(node.Address),
+		})
+	}
+	return resp, nil
+}
@@ -2,7 +2,6 @@ package weed_server

 import (
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/pb"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -12,6 +11,9 @@ import (
 	"sync"
 	"time"

+	"github.com/chrislusf/seaweedfs/weed/cluster"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+
 	"github.com/chrislusf/raft"
 	"github.com/gorilla/mux"
 	"google.golang.org/grpc"
@@ -60,13 +62,15 @@ type MasterServer struct {

 	// notifying clients
 	clientChansLock sync.RWMutex
-	clientChans     map[string]chan *master_pb.VolumeLocation
+	clientChans     map[string]chan *master_pb.KeepConnectedResponse

 	grpcDialOption grpc.DialOption

 	MasterClient *wdclient.MasterClient

 	adminLocks *AdminLocks
+
+	Cluster *cluster.Cluster
 }

 func NewMasterServer(r *mux.Router, option *MasterOption, peers []pb.ServerAddress) *MasterServer {
@@ -99,10 +103,11 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers []pb.ServerAddre
 		option:          option,
 		preallocateSize: preallocateSize,
 		vgCh:            make(chan *topology.VolumeGrowRequest, 1<<6),
-		clientChans:     make(map[string]chan *master_pb.VolumeLocation),
+		clientChans:     make(map[string]chan *master_pb.KeepConnectedResponse),
 		grpcDialOption:  grpcDialOption,
-		MasterClient:    wdclient.NewMasterClient(grpcDialOption, "master", option.Master, "", peers),
+		MasterClient:    wdclient.NewMasterClient(grpcDialOption, cluster.MasterType, option.Master, "", peers),
 		adminLocks:      NewAdminLocks(),
+		Cluster:         cluster.NewCluster(),
 	}
 	ms.boundedLeaderChan = make(chan int, 16)

@@ -206,10 +211,20 @@ func (ms *MasterServer) startAdminScripts() {

 	v := util.GetViper()
 	adminScripts := v.GetString("master.maintenance.scripts")
-	glog.V(0).Infof("adminScripts:\n%v", adminScripts)
 	if adminScripts == "" {
-		return
+		adminScripts = `
+		lock
+		ec.encode -fullPercent=95 -quietFor=1h
+		ec.rebuild -force
+		ec.balance -force
+		volume.deleteEmpty -quietFor=24h -force
+		volume.balance -force
+		volume.fix.replication
+		s3.clean.uploads -timeAgo=24h
+		unlock
+	`
 	}
+	glog.V(0).Infof("adminScripts: %v", adminScripts)

 	v.SetDefault("master.maintenance.sleep_minutes", 17)
 	sleepMinutes := v.GetInt("master.maintenance.sleep_minutes")
@@ -22,7 +22,7 @@ import (
 const BufferSizeLimit = 1024 * 1024 * 2

 // VolumeCopy copy the .idx .dat .vif files, and mount the volume
-func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.VolumeCopyRequest) (*volume_server_pb.VolumeCopyResponse, error) {
+func (vs *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, stream volume_server_pb.VolumeServer_VolumeCopyServer) error {

 	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
 	if v != nil {
@@ -31,7 +31,7 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo

 		err := vs.store.DeleteVolume(needle.VolumeId(req.VolumeId))
 		if err != nil {
-			return nil, fmt.Errorf("failed to delete existing volume %d: %v", req.VolumeId, err)
+			return fmt.Errorf("failed to delete existing volume %d: %v", req.VolumeId, err)
 		}

 		glog.V(0).Infof("deleted existing volume %d before copying.", req.VolumeId)
@@ -79,22 +79,38 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo
 		}()

 		// println("source:", volFileInfoResp.String())
+		copyResponse := &volume_server_pb.VolumeCopyResponse{}
+		reportInterval := int64(1024 * 1024 * 128)
+		nextReportTarget := reportInterval
 		var modifiedTsNs int64
-		if modifiedTsNs, err = vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, dataBaseFileName, ".dat", false, true); err != nil {
+		var sendErr error
+		if modifiedTsNs, err = vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, dataBaseFileName, ".dat", false, true, func(processed int64) bool {
+			if processed > nextReportTarget {
+				copyResponse.ProcessedBytes = processed
+				if sendErr = stream.Send(copyResponse); sendErr != nil {
+					return false
+				}
+				nextReportTarget = processed + reportInterval
+			}
+			return true
+		}); err != nil {
 			return err
 		}
+		if sendErr != nil {
+			return sendErr
+		}
 		if modifiedTsNs > 0 {
 			os.Chtimes(dataBaseFileName+".dat", time.Unix(0, modifiedTsNs), time.Unix(0, modifiedTsNs))
 		}

-		if modifiedTsNs, err = vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.IdxFileSize, indexBaseFileName, ".idx", false, false); err != nil {
+		if modifiedTsNs, err = vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.IdxFileSize, indexBaseFileName, ".idx", false, false, nil); err != nil {
 			return err
 		}
 		if modifiedTsNs > 0 {
 			os.Chtimes(indexBaseFileName+".idx", time.Unix(0, modifiedTsNs), time.Unix(0, modifiedTsNs))
 		}

-		if modifiedTsNs, err = vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, dataBaseFileName, ".vif", false, true); err != nil {
+		if modifiedTsNs, err = vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, dataBaseFileName, ".vif", false, true, nil); err != nil {
 			return err
 		}
 		if modifiedTsNs > 0 {
@@ -107,10 +123,10 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo
 	})

 	if err != nil {
-		return nil, err
+		return err
 	}
 	if dataBaseFileName == "" {
-		return nil, fmt.Errorf("not found volume %d file", req.VolumeId)
+		return fmt.Errorf("not found volume %d file", req.VolumeId)
 	}

 	idxFileName = indexBaseFileName + ".idx"
@@ -125,21 +141,25 @@ func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.Vo
 	}()

 	if err = checkCopyFiles(volFileInfoResp, idxFileName, datFileName); err != nil { // added by panyc16
-		return nil, err
+		return err
 	}

 	// mount the volume
 	err = vs.store.MountVolume(needle.VolumeId(req.VolumeId))
 	if err != nil {
-		return nil, fmt.Errorf("failed to mount volume %d: %v", req.VolumeId, err)
+		return fmt.Errorf("failed to mount volume %d: %v", req.VolumeId, err)
 	}

-	return &volume_server_pb.VolumeCopyResponse{
+	if err = stream.Send(&volume_server_pb.VolumeCopyResponse{
 		LastAppendAtNs: volFileInfoResp.DatFileTimestampSeconds * uint64(time.Second),
-	}, err
+	}); err != nil {
+		glog.Errorf("send response: %v", err)
+	}
+
+	return err
 }

-func (vs *VolumeServer) doCopyFile(client volume_server_pb.VolumeServerClient, isEcVolume bool, collection string, vid, compactRevision uint32, stopOffset uint64, baseFileName, ext string, isAppend, ignoreSourceFileNotFound bool) (modifiedTsNs int64, err error) {
+func (vs *VolumeServer) doCopyFile(client volume_server_pb.VolumeServerClient, isEcVolume bool, collection string, vid, compactRevision uint32, stopOffset uint64, baseFileName, ext string, isAppend, ignoreSourceFileNotFound bool, progressFn storage.ProgressFunc) (modifiedTsNs int64, err error) {

 	copyFileClient, err := client.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
 		VolumeId:                 vid,
@@ -154,7 +174,7 @@ func (vs *VolumeServer) doCopyFile(client volume_server_pb.VolumeServerClient, i
 		return modifiedTsNs, fmt.Errorf("failed to start copying volume %d %s file: %v", vid, ext, err)
 	}

-	modifiedTsNs, err = writeToFile(copyFileClient, baseFileName+ext, util.NewWriteThrottler(vs.compactionBytePerSecond), isAppend)
+	modifiedTsNs, err = writeToFile(copyFileClient, baseFileName+ext, util.NewWriteThrottler(vs.compactionBytePerSecond), isAppend, progressFn)
 	if err != nil {
 		return modifiedTsNs, fmt.Errorf("failed to copy %s file: %v", baseFileName+ext, err)
 	}
@@ -188,7 +208,7 @@ func checkCopyFiles(originFileInf *volume_server_pb.ReadVolumeFileStatusResponse
 	return nil
 }

-func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName string, wt *util.WriteThrottler, isAppend bool) (modifiedTsNs int64, err error) {
+func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName string, wt *util.WriteThrottler, isAppend bool, progressFn storage.ProgressFunc) (modifiedTsNs int64, err error) {
 	glog.V(4).Infof("writing to %s", fileName)
 	flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
 	if isAppend {
@@ -200,18 +220,25 @@ func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName s
 	}
 	defer dst.Close()

+	var progressedBytes int64
 	for {
 		resp, receiveErr := client.Recv()
 		if receiveErr == io.EOF {
 			break
 		}
-		if resp.ModifiedTsNs != 0 {
+		if resp!=nil && resp.ModifiedTsNs != 0 {
 			modifiedTsNs = resp.ModifiedTsNs
 		}
 		if receiveErr != nil {
 			return modifiedTsNs, fmt.Errorf("receiving %s: %v", fileName, receiveErr)
 		}
 		dst.Write(resp.FileContent)
+		progressedBytes += int64(len(resp.FileContent))
+		if progressFn != nil {
+			if !progressFn(progressedBytes) {
+				return modifiedTsNs, fmt.Errorf("interrupted copy operation")
+			}
+		}
 		wt.MaybeSlowdown(int64(len(resp.FileContent)))
 	}
 	return modifiedTsNs, nil
@@ -130,7 +130,7 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv

 		// copy ec data slices
 		for _, shardId := range req.ShardIds {
-			if _, err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, erasure_coding.ToExt(int(shardId)), false, false); err != nil {
+			if _, err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, erasure_coding.ToExt(int(shardId)), false, false, nil); err != nil {
 				return err
 			}
 		}
@@ -138,7 +138,7 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv
 		if req.CopyEcxFile {

 			// copy ecx file
-			if _, err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, indexBaseFileName, ".ecx", false, false); err != nil {
+			if _, err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, indexBaseFileName, ".ecx", false, false, nil); err != nil {
 				return err
 			}
 			return nil
@@ -146,14 +146,14 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv

 		if req.CopyEcjFile {
 			// copy ecj file
-			if _, err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, indexBaseFileName, ".ecj", true, true); err != nil {
+			if _, err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, indexBaseFileName, ".ecj", true, true, nil); err != nil {
 				return err
 			}
 		}

 		if req.CopyVifFile {
 			// copy vif file
-			if _, err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, ".vif", false, true); err != nil {
+			if _, err := vs.doCopyFile(client, true, req.Collection, req.VolumeId, math.MaxUint32, math.MaxInt64, dataBaseFileName, ".vif", false, true, nil); err != nil {
 				return err
 			}
 		}
@@ -62,13 +62,8 @@ func (vs *VolumeServer) VolumeTierMoveDatToRemote(req *volume_server_pb.VolumeTi
 		})
 	}

-	// remember the file original source
-	attributes := make(map[string]string)
-	attributes["volumeId"] = v.Id.String()
-	attributes["collection"] = v.Collection
-	attributes["ext"] = ".dat"
 	// copy the data file
-	key, size, err := backendStorage.CopyFile(diskFile.File, attributes, fn)
+	key, size, err := backendStorage.CopyFile(diskFile.File, fn)
 	if err != nil {
 		return fmt.Errorf("backend %s copy file %s: %v", req.DestinationBackendName, diskFile.Name(), err)
 	}
@@ -24,19 +24,35 @@ func (vs *VolumeServer) VacuumVolumeCheck(ctx context.Context, req *volume_serve

 }

-func (vs *VolumeServer) VacuumVolumeCompact(ctx context.Context, req *volume_server_pb.VacuumVolumeCompactRequest) (*volume_server_pb.VacuumVolumeCompactResponse, error) {
+func (vs *VolumeServer) VacuumVolumeCompact(req *volume_server_pb.VacuumVolumeCompactRequest, stream volume_server_pb.VolumeServer_VacuumVolumeCompactServer) error {

 	resp := &volume_server_pb.VacuumVolumeCompactResponse{}
+	reportInterval := int64(1024 * 1024 * 128)
+	nextReportTarget := reportInterval

-	err := vs.store.CompactVolume(needle.VolumeId(req.VolumeId), req.Preallocate, vs.compactionBytePerSecond)
+	var sendErr error
+	err := vs.store.CompactVolume(needle.VolumeId(req.VolumeId), req.Preallocate, vs.compactionBytePerSecond, func(processed int64) bool {
+		if processed > nextReportTarget {
+			resp.ProcessedBytes = processed
+			if sendErr = stream.Send(resp); sendErr != nil {
+				return false
+			}
+			nextReportTarget = processed + reportInterval
+		}
+		return true
+	})

 	if err != nil {
 		glog.Errorf("compact volume %d: %v", req.VolumeId, err)
-	} else {
-		glog.V(1).Infof("compact volume %d", req.VolumeId)
+		return err
+	}
+	if sendErr != nil {
+		glog.Errorf("compact volume %d report progress: %v", req.VolumeId, sendErr)
+		return sendErr
 	}

-	return resp, err
+	glog.V(1).Infof("compact volume %d", req.VolumeId)
+	return nil

 }

@@ -0,0 +1,55 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/cluster"
+	"io"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandClusterPs{})
+}
+
+type commandClusterPs struct {
+}
+
+func (c *commandClusterPs) Name() string {
+	return "cluster.ps"
+}
+
+func (c *commandClusterPs) Help() string {
+	return `check current cluster process status
+
+	cluster.ps
+
+`
+}
+
+func (c *commandClusterPs) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	clusterPsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	if err = clusterPsCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
+		resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
+			ClientType: cluster.FilerType,
+		})
+
+		fmt.Fprintf(writer, "the cluster has %d filers\n", len(resp.ClusterNodes))
+		for _, node := range resp.ClusterNodes {
+			fmt.Fprintf(writer, "  * %s (%v)\n", node.Address, node.Version)
+		}
+		return err
+	})
+	if err != nil {
+		return
+	}
+
+	return nil
+}
@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 	"io"
@@ -40,6 +41,7 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 	encodeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
 	volumeId := encodeCommand.Int("volumeId", 0, "the volume id")
 	collection := encodeCommand.String("collection", "", "the collection name")
+	forceChanges := encodeCommand.Bool("force", false, "force the encoding even if the cluster has less than recommended 4 nodes")
 	if err = encodeCommand.Parse(args); err != nil {
 		return nil
 	}
@@ -56,6 +58,17 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 		return err
 	}

+	if !*forceChanges {
+		var nodeCount int
+		eachDataNode(topologyInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
+			nodeCount++
+		})
+		if nodeCount < erasure_coding.ParityShardsCount {
+			glog.V(0).Infof("skip erasure coding with %d nodes, less than recommended %d nodes", nodeCount, erasure_coding.ParityShardsCount)
+			return nil
+		}
+	}
+
 	// volumeId is provided
 	if vid != 0 {
 		return doEcDecode(commandEnv, topologyInfo, *collection, vid)
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"io"
+	"math/rand"
 	"sync"
 	"time"

@@ -33,7 +34,7 @@ func (c *commandEcEncode) Name() string {
 func (c *commandEcEncode) Help() string {
 	return `apply erasure coding to a volume

-	ec.encode [-collection=""] [-fullPercent=95] [-quietFor=1h]
+	ec.encode [-collection=""] [-fullPercent=95 -quietFor=1h]
 	ec.encode [-collection=""] [-volumeId=<volume_id>]

 	This command will:
@@ -248,7 +249,7 @@ func parallelCopyEcShardsFromSource(grpcDialOption grpc.DialOption, targetServer
 func balancedEcDistribution(servers []*EcNode) (allocated [][]uint32) {
 	allocated = make([][]uint32, len(servers))
 	allocatedShardIdIndex := uint32(0)
-	serverIndex := 0
+	serverIndex := rand.Intn(len(servers))
 	for allocatedShardIdIndex < erasure_coding.TotalShardsCount {
 		if servers[serverIndex].freeEcSlot > 0 {
 			allocated[serverIndex] = append(allocated[serverIndex], allocatedShardIdIndex)
@@ -26,7 +26,7 @@ func (c *commandRemoteCache) Help() string {
 	return `cache the file content for mounted directories or files

 	# assume a remote storage is configured to name "cloud1"
-	remote.configure -name=cloud1 -type=s3 -access_key=xxx -secret_key=yyy
+	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
 	# mount and pull one bucket
 	remote.mount -dir=/xxx -remote=cloud1/bucket

@@ -163,10 +163,10 @@ func (c *commandRemoteCache) cacheContentData(commandEnv *CommandEnv, writer io.

 			remoteLocation := filer.MapFullPathToRemoteStorageLocation(localMountedDir, remoteMountedLocation, dir.Child(entry.Name))

-			if err := filer.DownloadToLocal(commandEnv, remoteConf, remoteLocation, dir, entry); err != nil {
-				fmt.Fprintf(writer, "DownloadToLocal %+v: %v\n", remoteLocation, err)
+			if err := filer.CacheRemoteObjectToLocalCluster(commandEnv, remoteConf, remoteLocation, dir, entry); err != nil {
+				fmt.Fprintf(writer, "CacheRemoteObjectToLocalCluster %+v: %v\n", remoteLocation, err)
 				if executionErr == nil {
-					executionErr = fmt.Errorf("DownloadToLocal %+v: %v\n", remoteLocation, err)
+					executionErr = fmt.Errorf("CacheRemoteObjectToLocalCluster %+v: %v\n", remoteLocation, err)
 				}
 				return
 			}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Chris Lu	7227cfddf5	2.80	2021-11-29 00:57:08 -08:00
Chris Lu	3d7390302d	add s3.clean.uploads -timeAgo=24h	2021-11-29 00:49:49 -08:00
Chris Lu	2f72c24498	skip the rest logic	2021-11-29 00:32:21 -08:00
Chris Lu	88ff8fc27b	ensure uploaded chunks are deleted on error	2021-11-29 00:28:26 -08:00
Chris Lu	bea26549f2	avoid empty extended attribute value	2021-11-28 23:50:52 -08:00
Chris Lu	ce2af0082e	revert	2021-11-28 23:35:22 -08:00
Chris Lu	1c9f3c7ac0	read deleted chunks when replcating data	2021-11-28 23:34:34 -08:00
Chris Lu	96c66ca2aa	read deleted chunks when replicating data	2021-11-28 23:33:03 -08:00
Chris Lu	f3a334965d	Update command_remote_uncache.go minor	2021-11-28 23:10:32 -08:00
Chris Lu	9ccfc1cfee	Update command_volume_fsck.go minor	2021-11-28 22:39:24 -08:00
Chris Lu	71a94267c9	Update command_volume_fsck.go remove unused code	2021-11-28 22:37:11 -08:00
Chris Lu	ad16221a35	adjust error log	2021-11-28 22:06:17 -08:00
Chris Lu	cf1586a34d	add logs for writing to remote file	2021-11-27 22:09:23 -08:00
Chris Lu	3a19eea97c	allocate memory by slabs	2021-11-27 12:13:00 -08:00
Chris LuandGitHub	e87f276cf6	Merge pull request #2470 from hilimd/master fix redis2 sentinel config example	2021-11-25 09:05:09 -08:00
limd	8805c04128	fix redis2 sentinel config example	2021-11-25 19:20:02 +08:00
Chris LuandGitHub	6f95c06bd5	Merge pull request #2469 from hilimd/master support redis2 sentinel	2021-11-25 00:15:46 -08:00
limd	ec03f22cc3	Merge remote-tracking branch 'origin/master'	2021-11-25 16:07:14 +08:00
limd	220797bd71	support redis sentinel	2021-11-25 15:57:03 +08:00
hilimdandGitHub	aa1ef02984	Merge pull request #88 from chrislusf/master sync	2021-11-25 15:53:35 +08:00
Chris Lu	da1c755f6a	fix build issue fix https://github.com/wangyoucao577/go-release-action/issues/66	2021-11-23 07:39:43 -08:00
Chris Lu	6c27845be0	add retries when writing to remote s3 fix https://github.com/chrislusf/seaweedfs/issues/2465	2021-11-22 21:48:04 -08:00
Chris LuandGitHub	9e2ae3f3b2	Merge pull request #2464 from zer0def/k8s-resource-apis Update K8S resource API versions in Helm charts	2021-11-21 22:23:09 -08:00
zer0def	01558a0bd5	Update resource API versions up to K8S 1.22	2021-11-22 07:11:53 +01:00
Chris Lu	f3c789d662	2.79	2021-11-21 18:40:24 -08:00
Chris LuandGitHub	9eebe9b95e	Merge pull request #2454 from kmlebedev/HeadBucketHandlerForbidden Avoid forbiddening for HeadBucketHandler if owners are specified	2021-11-17 08:37:05 -08:00
Konstantin Lebedev	40abae7caa	Avoid forbiddening for HeadBucketHandler if owners are specified https://github.com/chrislusf/seaweedfs/issues/2434	2021-11-17 14:42:24 +05:00
Chris Lu	3d87aa767d	fix same dc and other dc	2021-11-16 09:14:01 -08:00
Chris Lu	a0ef6e3611	prevent nil response fix https://github.com/chrislusf/seaweedfs/issues/2452	2021-11-15 08:46:01 -08:00
Chris Lu	791aa3cb44	update to wangyoucao577/go-release-action@v1.21	2021-11-15 08:33:46 -08:00
Chris Lu	320a08f250	adjust go release action version	2021-11-14 23:45:34 -08:00
Chris Lu	100c654ec3	2.78	2021-11-14 23:29:59 -08:00
Chris Lu	947fe4cc97	docker: entry point change working directory from root to /data	2021-11-12 17:21:49 -08:00
Chris Lu	7bf891c00a	randomize same-dc servers and other-dc servers	2021-11-12 11:30:11 -08:00
Chris Lu	1f75f1f9dc	filer: fix mysql2 SQL template	2021-11-11 22:28:28 -08:00
Chris Lu	3abbaccb70	filer: fix mysql command to upsert	2021-11-11 22:27:13 -08:00
Chris Lu	177129013c	releases in github actions disable http2client	2021-11-11 21:28:06 -08:00
Chris LuandGitHub	4d388b9782	Update binaries_dev.yml	2021-11-11 18:37:38 -08:00
Chris LuandGitHub	5eb905a5d1	Update binaries_dev.yml	2021-11-11 12:05:23 -08:00
Chris LuandGitHub	043a1337df	Merge pull request #2440 from wangyoucao577/bugfix/unexpeced-eof Draft: Fix go-release-action upload asset fail	2021-11-11 12:02:55 -08:00
Chris LuandGitHub	bef9c2943b	Merge branch 'master' into bugfix/unexpeced-eof	2021-11-11 12:02:42 -08:00
Chris LuandGitHub	79a1903021	Update binaries_dev.yml	2021-11-11 11:53:47 -08:00
Jay Zhang	12d9108bb4	chore: disable http2 in go-release-action	2021-11-11 19:59:12 +08:00
Chris LuandGitHub	cef2718ede	Merge pull request #2438 from ekhvalov/volume_replication_collection Volume configure replication by collection pattern	2021-11-11 01:20:21 -08:00
user	9668b15f38	Filtering by volume id is prioritized.	2021-11-11 18:18:56 +09:00
user	563a74a9eb	Volume filter by collection pattern added.	2021-11-11 18:01:47 +09:00
user	dbb8003ce3	Volume filter function added.	2021-11-11 17:45:17 +09:00
hilimdandGitHub	34240606f7	Merge pull request #87 from chrislusf/master sync	2021-11-11 16:00:21 +08:00
user	c387fe957b	Unused parameter removed.	2021-11-11 16:08:41 +09:00
Chris LuandGitHub	835e2d2ddf	Merge pull request #2435 from kmlebedev/delAmzIsAdmin force overwrite s3-is-admin	2021-11-10 22:39:39 -08:00
Konstantin Lebedev	42f4aa824b	force overwrite s3-is-admin https://github.com/chrislusf/seaweedfs/issues/2433	2021-11-11 11:34:01 +05:00
Chris LuandGitHub	b0b2a03ea7	Update binaries_dev.yml	2021-11-10 12:27:45 -08:00
Chris LuandGitHub	641ce36f6f	Update binaries_dev.yml	2021-11-10 09:58:54 -08:00
Chris LuandGitHub	22f75177bf	Update backers.md	2021-11-09 23:53:35 -08:00
Chris Lu	c77c0de418	make install	2021-11-09 12:34:57 -08:00
Chris LuandGitHub	a7eee8b14b	Merge pull request #2427 from Bl1tz23/master Fix: potencial iam identities data race	2021-11-09 08:53:48 -08:00
Chris Lu	1d4a61af5e	add brokers	2021-11-09 08:50:55 -08:00
Bl1tz23	c683409e92	s3: add RWMutex to iam, use RLock for concurrent reading	2021-11-09 18:11:06 +03:00
Bl1tz23	5018b22f36	s3: fix potencial iam identities data race	2021-11-09 12:19:50 +03:00
Chris Lu	59d1435d65	s3: avoid possible attacks by version	2021-11-08 17:47:56 -08:00
Chris Lu	5c99b8acc3	rename variables	2021-11-08 17:47:56 -08:00
Chris Lu	00fdab1284	adjust client name	2021-11-08 17:47:56 -08:00
Chris Lu	4729a57cc0	use constants	2021-11-08 17:47:56 -08:00
Chris Lu	d9dd72ea56	rename pacakge	2021-11-08 17:47:56 -08:00
Chris LuandGitHub	73a03cd965	Update backers.md	2021-11-08 09:25:08 -08:00
Chris Lu	5cf332357b	2.77	2021-11-07 13:52:45 -08:00
Chris Lu	244841f080	adjust logs	2021-11-07 13:27:57 -08:00
Chris Lu	6e999f1176	s3: fix ListObject if more than 10000 objects	2021-11-07 12:39:36 -08:00
Chris Lu	a8b0f8864d	fix help message	2021-11-07 12:38:35 -08:00
Chris Lu	55973e8572	log unknown access key	2021-11-07 12:37:46 -08:00
Chris Lu	c4e22b5a9a	filer: deprecate "-peers" option	2021-11-06 14:36:45 -07:00
Chris Lu	751a7073e3	Merge branch 'flexible_filer_addition'	2021-11-06 14:26:26 -07:00
Chris Lu	e0fc2898e9	auto updated filer peer list	2021-11-06 14:23:35 -07:00
Chris Lu	04663c3611	remote.mount: print out metadata sync errors	2021-11-06 11:29:50 -07:00
Chris Lu	330d1fde7f	send peers info to filers	2021-11-06 11:29:50 -07:00
Chris LuandGitHub	12e6692dac	trying wangyoucao577/go-release-action@bugfix/upload-fail following https://github.com/wangyoucao577/go-release-action/issues/63#issuecomment-962417798	2021-11-06 05:34:34 -07:00
Chris Lu	84bb8e7365	send peers info to filers	2021-11-06 04:07:38 -07:00
Chris Lu	4b9c42996a	refactor grpc API	2021-11-05 18:11:40 -07:00
Chris Lu	5ea86ef1da	Revert "master: rename grpc function KeepConnected() to SubscribeVolumeLocationUpdates()" This reverts commit `af71ae11aa`.	2021-11-05 17:52:15 -07:00
Chris Lu	314c32514b	add command to list filers	2021-11-05 17:39:41 -07:00
Chris Lu	7348d2ae63	randomize traffic if replicated	2021-11-05 00:03:13 -07:00
Chris Lu	00ae965d8d	randomize a bit for ec shards distribution	2021-11-04 09:23:40 -07:00
Chris Lu	b485c905d4	docker image: build with arm also fix https://github.com/chrislusf/seaweedfs/issues/2418	2021-11-04 08:57:24 -07:00
Chris Lu	77f90ae288	add leader election in master	2021-11-04 00:54:38 -07:00
Chris Lu	35c37562bc	check auth only when enabled	2021-11-03 12:11:36 -07:00
Chris Lu	3c245c69d3	handle possible race condition	2021-11-03 01:36:39 -07:00
Chris Lu	6fb6480a3b	Java: 1.7.0 update org.apache.httpcomponents to 4.5.13 update grpc API to use cacheRemoteObjectToLocalCluster	2021-11-03 01:36:26 -07:00
Chris Lu	af71ae11aa	master: rename grpc function KeepConnected() to SubscribeVolumeLocationUpdates()	2021-11-03 01:09:48 -07:00
Chris Lu	ab97b17e62	better printout	2021-11-02 23:45:47 -07:00
Chris Lu	e66865a8c5	adjust master client log level	2021-11-02 23:45:28 -07:00
Chris Lu	0c8dea9de8	go fmt	2021-11-02 23:39:16 -07:00
Chris Lu	5160eb08f7	shell: optionally read filer address from master	2021-11-02 23:38:45 -07:00
Chris Lu	18bfbf62fc	add one unit test	2021-11-02 23:36:43 -07:00
Chris Lu	e1ab8b01d0	s3: adjust permission for HEAD bucket operation fix https://github.com/chrislusf/seaweedfs/issues/2417#issuecomment-958391856	2021-11-02 16:28:55 -07:00
Chris Lu	b25661c6df	s3: skip permission checking for creating bucket if the bucket already exists fix https://github.com/chrislusf/seaweedfs/issues/2417 Rclone was trying to create the bucket even though the bucket already exists.	2021-11-02 13:13:36 -07:00
Chris Lu	1b90d60765	filer: paginate large directories to delete file chunks	2021-11-02 01:04:50 -07:00
Chris Lu	794375ca0a	adjust help message since both fullPercent and quietFor are needed.	2021-11-01 17:22:47 -07:00
Chris Lu	fc9e246592	2.76	2021-10-31 18:08:28 -07:00
Chris Lu	9cf756b188	S3: support CORS related to https://github.com/chrislusf/seaweedfs/issues/1271	2021-10-31 18:06:43 -07:00
Chris Lu	f17fa400d5	refactoring	2021-10-31 18:05:34 -07:00
Chris Lu	006b4ec0f8	refactoring	2021-10-31 18:02:08 -07:00
Chris Lu	edbf6d297b	filer.meta.tail: add example to send metadata to elastic search	2021-10-31 18:01:33 -07:00
Chris Lu	24858507cc	rename API to avoid confusion	2021-10-30 19:27:25 -07:00
Chris Lu	f5824f369c	avoid reporting error when it is a user request error	2021-10-29 14:14:29 -07:00
Chris Lu	c857cc7286	cloud tier: remove tagging since not all s3 vendors support this	2021-10-29 12:39:19 -07:00
Chris Lu	d04cdcf40d	s3 header add user agent	2021-10-29 12:28:24 -07:00
hilimdandGitHub	b5389c3b23	Merge pull request #86 from chrislusf/master sync	2021-10-29 23:06:23 +08:00
Chris LuandGitHub	608dc11a28	Merge pull request #2397 from kmlebedev/fix_DeleteBucketLifecycleConfiguration fix DeleteBucketLifecycleConfiguration	2021-10-28 08:16:56 -07:00
Konstantin Lebedev	dc3ae5e8a6	DeleteBucketLifecycleConfiguration return 204	2021-10-28 19:34:37 +05:00
Konstantin Lebedev	2afb5a13af	fix DeleteBucketLifecycleConfiguration	2021-10-28 18:30:33 +05:00
hilimdandGitHub	9e3060bb13	Merge pull request #85 from chrislusf/master sync	2021-10-28 15:08:22 +08:00
Chris Lu	2e76834e4d	filer store: redis2 fix wrong pagination	2021-10-27 23:46:25 -07:00
Chris Lu	54b6e0f3fd	adjust logs	2021-10-27 23:46:07 -07:00
Chris Lu	900e888695	filer: avoid possible prefixed pagination loop on unsupported filer stores	2021-10-27 23:45:48 -07:00
hilimdandGitHub	6999325d36	Merge pull request #84 from chrislusf/master sync	2021-10-27 09:42:23 +08:00
Chris Lu	3eeaffeadd	update gjson resolve github.com/tidwall/gjson vulnerability found	2021-10-25 16:13:31 -07:00
Chris Lu	d774fa6c9a	rename variable	2021-10-25 14:39:20 -07:00
Chris Lu	2539ba0b62	fix compilation	2021-10-25 14:38:48 -07:00
Chris Lu	5f2d7c1589	erasure coding: skip erasure coding if less than recommended 4 nodes	2021-10-25 14:38:11 -07:00
Chris Lu	5296c397f4	build latest container	2021-10-24 18:45:51 -07:00
Chris Lu	c9d3fb4a30	2.75	2021-10-24 18:15:59 -07:00
Chris Lu	5435027ff0	volume copy: stream out copying progress and avoid grpc request timeout fix https://github.com/chrislusf/seaweedfs/issues/2386	2021-10-24 02:52:56 -07:00
Chris Lu	3be3c17f59	volume vacuum: avoid timeout with streaming progress report fix https://github.com/chrislusf/seaweedfs/issues/2396	2021-10-24 01:55:34 -07:00
hilimdandGitHub	ee90edd0e3	Merge pull request #83 from chrislusf/master sync	2021-10-23 19:52:48 +08:00
Chris LuandGitHub	07dd4873db	Merge pull request #2390 from kmlebedev/fix_tagging	2021-10-20 10:59:22 -07:00
Konstantin Lebedev	235329a92a	fix Tagging add xmlns	2021-10-20 19:40:22 +05:00
Konstantin Lebedev	dc7e525cb9	fix Tagging test	2021-10-20 19:12:00 +05:00
Konstantin Lebedev	0b834600d5	fix PutObjectTaggingHandler Unmarshal	2021-10-20 17:58:06 +05:00
Konstantin Lebedev	c78220a7f2	fix object tagging https://github.com/chrislusf/seaweedfs/issues/2389	2021-10-20 16:01:06 +05:00
Chris Lu	182f43ae5f	2.74	2021-10-18 14:23:54 -07:00
Chris Lu	309f46e3af	S3: fix upload limit if the size is more than 8GB	2021-10-18 14:14:18 -07:00
Chris Lu	cd4fa7561b	2.73	2021-10-18 10:47:48 -07:00
Chris Lu	a6bb509cb7	adjust mime detection	2021-10-18 10:47:39 -07:00
Chris LuandGitHub	825d76622b	Merge pull request #2388 from kmlebedev/fix_mime fix mine detect	2021-10-18 10:23:23 -07:00
Konstantin Lebedev	dd2cc1acaf	fix https://github.com/chrislusf/seaweedfs/issues/2387	2021-10-18 16:27:57 +05:00
Chris Lu	8cd7a0365b	1.6.9	2021-10-17 18:41:23 -07:00
hilimdandGitHub	0548ed3a1b	Merge pull request #82 from chrislusf/master sync	2021-10-11 16:35:47 +08:00
hilimdandGitHub	1de733fda5	Merge pull request #81 from chrislusf/master sync	2021-09-13 10:34:33 +08:00
hilimdandGitHub	27c05f8c0b	Merge pull request #80 from chrislusf/master sync	2021-08-16 00:54:51 +08:00
hilimdandGitHub	97ad3e9d02	Merge pull request #79 from chrislusf/master sync	2021-06-06 10:13:14 +08:00
hilimdandGitHub	ff26e64648	Merge pull request #78 from chrislusf/master sync	2021-05-27 15:48:38 +08:00
hilimdandGitHub	9524b08bec	Merge pull request #77 from chrislusf/master sync	2021-05-16 10:32:02 +08:00
hilimdandGitHub	aabea37af1	Merge pull request #76 from chrislusf/master sync	2021-05-08 13:46:12 +08:00