Merge branch 'master' of https://github.com/chrislusf/seaweedfs

fix manifest batch size
Merge pull request #2729 from banjiaojuhao/filer_metadata-resolve-manifest
2026-10-02 04:32:42 +00:00 · 2022-03-07 00:08:09 -08:00 · 2022-03-07 00:07:53 -08:00 · 2022-03-07 00:05:46 -08:00 · 2022-03-07 00:04:59 -08:00 · 2022-03-07 00:02:09 -08:00
502 changed files with 26569 additions and 15726 deletions
@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: chrislusf
+patreon: seaweedfs
@@ -6,57 +6,110 @@ on:

 jobs:

-
-  build-latest-docker-image:
-    runs-on: [ubuntu-latest]
+  cleanup:
+    runs-on: ubuntu-latest

    steps:
-      -
-        name: Checkout
+
+      - name: Delete old release assets
+        uses: mknejp/delete-release-assets@v1
+        with:
+          token: ${{ github.token }}
+          tag: dev
+          fail-if-no-assets: false
+          assets: |
+            weed-*
+
+  build_dev_linux_windows:
+    needs: cleanup
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, windows]
+        goarch: [amd64]
+
+    steps:
+
+      - name: Check out code into the Go module directory
        uses: actions/checkout@v2
-      -
-        name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v3
+
+      - name: Set BUILD_TIME env
+        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
+
+      - name: Go Release Binaries Large Disk
+        uses: wangyoucao577/go-release-action@v1.22
        with:
-          images: |
-            chrislusf/seaweedfs
-            ghcr.io/chrislusf/seaweedfs
-          tags: |
-            type=raw,value=latest
-          labels: |
-            org.opencontainers.image.title=seaweedfs
-            org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          release_tag: dev
+          overwrite: true
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed-large-disk
+          asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
+
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@v1.22
        with:
-          buildkitd-flags: "--debug"
-      -
-        name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          release_tag: dev
+          overwrite: true
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed-normal-disk
+          asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
+
+  build_dev_darwin:
+    needs: build_dev_linux_windows
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [darwin]
+        goarch: [amd64, arm64]
+
+    steps:
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Set BUILD_TIME env
+        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
+
+      - name: Go Release Binaries Large Disk
+        uses: wangyoucao577/go-release-action@v1.22
        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          release_tag: dev
+          overwrite: true
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed-large-disk
+          asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
+
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@v1.22
        with:
-          registry: ghcr.io
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
-          file: ./docker/Dockerfile
-          platforms: linux/amd64, linux/arm, linux/arm64
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          release_tag: dev
+          overwrite: true
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed-normal-disk
+          asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
@@ -0,0 +1,54 @@
+# This is a basic workflow to help you get started with Actions
+
+name: "go: build versioned binaries for windows"
+
+on:
+  push:
+    tags:
+      - '*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+
+  build-release-binaries_windows:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [windows]
+        goarch: [amd64]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@v1.22
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
+          # build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
+      - name: Go Release Large Disk Binaries
+        uses: wangyoucao577/go-release-action@v1.22
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"
@@ -1,6 +1,6 @@
 # This is a basic workflow to help you get started with Actions

-name: "go: build versioned binaries"
+name: "go: build versioned binaries for linux"

 on:
  push:
@@ -13,34 +13,25 @@ on:
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:

-  build-release-binaries:
+  build-release-binaries_linux:
    runs-on: ubuntu-latest
    strategy:
      matrix:
-        goos: [linux, windows, darwin, freebsd]
+        goos: [linux]
        goarch: [amd64, arm, arm64]
-        exclude:
-          - goarch: arm
-            goos: darwin
-          - goarch: 386
-            goos: darwin
-          - goarch: arm
-            goos: windows
-          - goarch: arm64
-            goos: windows

    # Steps represent a sequence of tasks that will be executed as part of the job
    steps:
      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
      - uses: actions/checkout@v2
      - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          # build_flags: -tags 5BytesOffset # optional, default is
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
@@ -48,13 +39,13 @@ jobs:
          binary_name: weed
          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
      - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.20
+        uses: wangyoucao577/go-release-action@v1.22
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          goos: ${{ matrix.goos }}
          goarch: ${{ matrix.goarch }}
          overwrite: true
-          pre_command: export CGO_ENABLED=0
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
          build_flags: -tags 5BytesOffset # optional, default is
          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
          # Where to run `go build .`
@@ -0,0 +1,54 @@
+# This is a basic workflow to help you get started with Actions
+
+name: "go: build versioned binaries for darwin"
+
+on:
+  push:
+    tags:
+      - '*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+
+  build-release-binaries_darwin:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [darwin]
+        goarch: [amd64, arm64]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@v1.22
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
+          # build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
+      - name: Go Release Large Disk Binaries
+        uses: wangyoucao577/go-release-action@v1.22
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"
@@ -0,0 +1,54 @@
+# This is a basic workflow to help you get started with Actions
+
+name: "go: build versioned binaries for freebsd"
+
+on:
+  push:
+    tags:
+      - '*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+
+  build-release-binaries_freebsd:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [freebsd]
+        goarch: [amd64, arm, arm64]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@v1.22
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
+          # build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
+      - name: Go Release Large Disk Binaries
+        uses: wangyoucao577/go-release-action@v1.22
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"
@@ -58,6 +58,6 @@ jobs:
          context: ./docker
          push: ${{ github.event_name != 'pull_request' }}
          file: ./docker/Dockerfile.go_build
-          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
+          platforms: linux/amd64, linux/arm64
          tags: ${{ steps.docker_meta.outputs.tags }}
          labels: ${{ steps.docker_meta.outputs.labels }}
@@ -2,7 +2,8 @@ name: "docker: build latest container"

 on:
  push:
-    branches: [ master ]
+    tags:
+      - '*'
  workflow_dispatch: []

 jobs:
@@ -23,7 +24,7 @@ jobs:
            chrislusf/seaweedfs
            ghcr.io/chrislusf/seaweedfs
          tags: |
-            type=raw,value=dev
+            type=raw,value=latest
          labels: |
            org.opencontainers.image.title=seaweedfs
            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
@@ -1,121 +0,0 @@
-name: "docker: build release containers"
-
-on:
-  push:
-    tags:
-      - '*'
-  workflow_dispatch: []
-
-jobs:
-  build-default-release-container:
-    runs-on: [ubuntu-latest]
-
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            chrislusf/seaweedfs
-            ghcr.io/chrislusf/seaweedfs
-          tags: |
-            type=ref,event=tag
-          flavor: |
-            latest=false
-          labels: |
-            org.opencontainers.image.title=seaweedfs
-            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
-            org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
-      -
-        name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
-          file: ./docker/Dockerfile.go_build
-          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
-  build-large-release-container:
-    runs-on: [ubuntu-latest]
-
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            chrislusf/seaweedfs
-            ghcr.io/chrislusf/seaweedfs
-          tags: |
-            type=ref,event=tag,suffix=_large_disk
-          flavor: |
-            latest=false
-          labels: |
-            org.opencontainers.image.title=seaweedfs
-            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
-            org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
-      -
-        name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
-          file: ./docker/Dockerfile.go_build_large
-          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
@@ -0,0 +1,54 @@
+name: "docker: build release containers for normal volume"
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch: []
+
+jobs:
+  build-default-release-container:
+    runs-on: [ubuntu-latest]
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            chrislusf/seaweedfs
+          tags: |
+            type=ref,event=tag
+          flavor: |
+            latest=false
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.go_build
+          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
@@ -0,0 +1,55 @@
+name: "docker: build release containers for large volume"
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch: []
+
+jobs:
+
+  build-large-release-container:
+    runs-on: [ubuntu-latest]
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            chrislusf/seaweedfs
+          tags: |
+            type=ref,event=tag,suffix=_large_disk
+          flavor: |
+            latest=false
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.go_build_large
+          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
@@ -0,0 +1,55 @@
+name: "docker: build release containers for rocksdb"
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch: []
+
+jobs:
+
+  build-large-release-container_rocksdb:
+    runs-on: [ubuntu-latest]
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            chrislusf/seaweedfs
+          tags: |
+            type=ref,event=tag,suffix=_large_disk_rocksdb
+          flavor: |
+            latest=false
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.rocksdb_large
+          platforms: linux/amd64
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
@@ -41,6 +41,7 @@ Your support will be really appreciated by me and other supporters!
 - [Download Binaries for different platforms](https://github.com/chrislusf/seaweedfs/releases/latest)
 - [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
 - [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
+- [SeaweedFS on Telegram](https://t.me/Seaweedfs) 
 - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
 - [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
@@ -51,6 +52,8 @@ Table of Contents
 =================

 * [Quick Start](#quick-start)
+    * [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
+    * [Quick Start with Single Binary](#quick-start-with-single-binary)
 * [Introduction](#introduction)
 * [Features](#features)
    * [Additional Features](#additional-features)
@@ -73,7 +76,7 @@ Table of Contents

 `docker run -p 8333:8333 chrislusf/seaweedfs server -s3`

-## Quick Start with single binary ##
+## Quick Start with Single Binary ##
 * Download the latest binary from https://github.com/chrislusf/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`
 * Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.

@@ -147,6 +150,7 @@ Faster and Cheaper than direct cloud storage!
 * [AES256-GCM Encrypted Storage][FilerDataEncryption] safely stores the encrypted data.
 * [Super Large Files][SuperLargeFiles] stores large or super large files in tens of TB.
 * [Cloud Drive][CloudDrive] mounts cloud storage to local cluster, cached for fast read and write with asynchronous write back.
+* [Gateway to Remote Object Store][GatewayToRemoteObjectStore] mirrors bucket operations to remote object storage, in addition to [Cloud Drive][CloudDrive]

 ## Kubernetes ##
 * [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
@@ -170,6 +174,7 @@ Faster and Cheaper than direct cloud storage!
 [FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication
 [KeyLargeValueStore]: https://github.com/chrislusf/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
 [CloudDrive]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Drive-Architecture
+[GatewayToRemoteObjectStore]: https://github.com/chrislusf/seaweedfs/wiki/Gateway-to-Remote-Object-Storage


 [Back to TOC](#table-of-contents)
@@ -499,7 +504,7 @@ git clone https://github.com/chrislusf/seaweedfs.git
 Step 3: download, compile, and install the project by executing the following command

 ```bash
-make install
+cd seaweedfs/weed && make install
 ```

 Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
@@ -5,12 +5,16 @@

 <h2 align="center">Generous Backers ($50+)</h2>

- [4Sight Imaging](https://www.4sightimaging.com/)
 - [Evercam Camera Management Software](https://evercam.io/)
- [Admiral](https://getadmiral.com)

 <h2 align="center">Backers</h2>

 - [ColorfulClouds Tech Co. Ltd.](https://caiyunai.com/)
 - [Haravan - Ecommerce Platform](https://www.haravan.com)
 - PeterCxy - Creator of Shelter App
+- [Hive Games](https://playhive.com/)
+- Flowm
+- Yoni Nakache
+- Catalin Constantin
+- MingLi Yuan
+- Leroy van Logchem
@@ -1,56 +0,0 @@
-FROM alpine
-
-# 'latest' or 'dev'
-ARG RELEASE=latest
-
-RUN \
-    ARCH=$(if [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "64" ]; then echo "amd64"; \
-         elif [ $(uname -m) == "x86_64" ] && [ $(getconf LONG_BIT) == "32" ]; then echo "386"; \
-         elif [ $(uname -m) == "aarch64" ]; then echo "arm64"; \
-         elif [ $(uname -m) == "armv7l" ]; then echo "arm"; \
-         elif [ $(uname -m) == "armv6l" ]; then echo "arm"; \
-         elif [ $(uname -m) == "s390x" ]; then echo "s390x"; \
-         elif [ $(uname -m) == "ppc64le" ]; then echo "ppc64le"; fi;) && \
-    echo "Building for $ARCH" 1>&2 && \
-    SUPERCRONIC_SHA1SUM=$(echo $ARCH | sed 's/386/e0126b0102b9f388ecd55714358e3ad60d0cebdb/g' | sed 's/amd64/5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85/g' | sed 's/arm64/e2714c43e7781bf1579c85aa61259245f56dbba1/g' | sed 's/arm/47481c3341bc3a1ae91a728e0cc63c8e6d3791ad/g') && \
-    SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.9/supercronic-linux-$ARCH && \
-    SUPERCRONIC=supercronic-linux-$ARCH && \
-    # Install SeaweedFS and Supercronic ( for cron job mode )
-    apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
-    apk add fuse && \
-    wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/${RELEASE} | egrep -o "chrislusf/seaweedfs/releases/download/.*/linux_$ARCH.tar.gz" | head -n 1) && \
-    tar -C /usr/bin/ -xzvf /tmp/linux_$ARCH.tar.gz && \
-    curl -fsSLO "$SUPERCRONIC_URL" && \
-    echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
-    chmod +x "$SUPERCRONIC" && \
-    mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
-    ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
-    apk del build-dependencies && \
-    rm -rf /tmp/*
-
-# volume server gprc port
-EXPOSE 18080
-# volume server http port
-EXPOSE 8080
-# filer server gprc port
-EXPOSE 18888
-# filer server http port
-EXPOSE 8888
-# master server shared gprc port
-EXPOSE 19333
-# master server shared http port
-EXPOSE 9333
-# s3 server http port
-EXPOSE 8333
-# webdav server http port
-EXPOSE 7333
-
-RUN mkdir -p /data/filerldb2
-
-VOLUME /data
-
-COPY filer.toml /etc/seaweedfs/filer.toml
-COPY entrypoint.sh /entrypoint.sh
-RUN chmod +x /entrypoint.sh
-
-ENTRYPOINT ["/entrypoint.sh"]
@@ -37,6 +37,7 @@ EXPOSE 7333
 RUN mkdir -p /data/filerldb2

 VOLUME /data
+WORKDIR /data

 RUN chmod +x /entrypoint.sh

@@ -1,4 +1,4 @@
-FROM amd64/golang:1.17-alpine as builder
+FROM golang:1.17-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
@@ -36,6 +36,7 @@ EXPOSE 7333
 RUN mkdir -p /data/filerldb2

 VOLUME /data
+WORKDIR /data

 RUN chmod +x /entrypoint.sh

@@ -1,4 +1,4 @@
-FROM amd64/golang:1.17-alpine as builder
+FROM golang:1.17-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
@@ -36,6 +36,7 @@ EXPOSE 7333
 RUN mkdir -p /data/filerldb2

 VOLUME /data
+WORKDIR /data

 RUN chmod +x /entrypoint.sh

@@ -26,6 +26,7 @@ EXPOSE 7333
 RUN mkdir -p /data/filerldb2

 VOLUME /data
+WORKDIR /data

 RUN chmod +x /entrypoint.sh

@@ -0,0 +1,61 @@
+FROM golang:1.17-buster as builder
+
+RUN apt-get update
+RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
+
+ENV ROCKSDB_VERSION v6.22.1
+
+# build RocksDB
+RUN cd /tmp && \
+    git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
+    cd rocksdb && \
+    PORTABLE=1 make static_lib && \
+    make install-static
+
+ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
+ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
+
+# build SeaweedFS
+RUN mkdir -p /go/src/github.com/chrislusf/
+RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
+ARG BRANCH=${BRANCH:-master}
+RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
+RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
+  && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
+  && go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"
+
+
+FROM alpine AS final
+LABEL author="Chris Lu"
+COPY --from=builder /go/bin/weed /usr/bin/
+RUN mkdir -p /etc/seaweedfs
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+RUN apk add fuse snappy gflags
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+# webdav server http port
+EXPOSE 7333
+
+RUN mkdir -p /data/filer_rocksdb
+
+VOLUME /data
+
+WORKDIR /data
+
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
@@ -13,6 +13,12 @@ build: binary
 	docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
 	rm ./weed

+build_gorocksdb:
+	docker build --no-cache -t chrislusf/gorocksdb -f Dockerfile.go_rocksdb .
+
+build_rocksdb:
+	docker build --no-cache -t chrislusf/seaweedfs:rocksdb -f Dockerfile.rocksdb_large .
+
 s3tests_build:
 	docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .

@@ -40,6 +46,9 @@ dev_registry: build
 dev_replicate: build
 	docker-compose -f compose/local-replicate-compose.yml -p seaweedfs up

+dev_auditlog: build
+	docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up
+
 cluster: build
 	docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up

@@ -0,0 +1,4 @@
+{
+    "fluent_port": 24224,
+    "fluent_host": "fluent"
+}
@@ -0,0 +1,36 @@
+version: '2'
+
+services:
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8333:8333
+      - 9333:9333
+      - 19333:19333
+      - 8084:8080
+      - 18084:18080
+      - 8888:8888
+      - 18888:18888
+      - 8000:8000
+    command: "server -ip=s3 -filer -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.auditLogConfig=/etc/seaweedfs/fluent.json -volume.max=0 -master.volumeSizeLimitMB=8 -volume.preStopSeconds=1"
+    volumes:
+      - ./fluent.json:/etc/seaweedfs/fluent.json
+      - ./s3.json:/etc/seaweedfs/s3.json
+    depends_on:
+      - fluent
+  fluent:
+    image: fluent/fluentd:v1.14
+    ports:
+      - 24224:24224
+  #s3tests:
+  #  image: chrislusf/ceph-s3-tests:local
+  #  volumes:
+  #    - ./s3tests.conf:/opt/s3-tests/s3tests.conf
+  #  environment:
+  #    S3TEST_CONF: "s3tests.conf"
+  #    NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
+  #    NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
+  #    NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
+  #  depends_on:
+  #    - s3
+  #    - fluent
@@ -6,25 +6,37 @@ services:
    ports:
      - 9333:9333
      - 19333:19333
-    command: "master -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m1"
+    command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
  master1:
    image: chrislusf/seaweedfs:local
    ports:
      - 9334:9334
      - 19334:19334
-    command: "master -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m2"
+    command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
  master2:
    image: chrislusf/seaweedfs:local
    ports:
      - 9335:9335
      - 19335:19335
-    command: "master -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/data/m3"
+    command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
  volume1:
    image: chrislusf/seaweedfs:local
    ports:
      - 8080:8080
      - 18080:18080
-    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1 -disk=ssd1'
+    command: 'volume -dataCenter dc1 -rack v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
    depends_on:
      - master0
      - master1
@@ -34,7 +46,7 @@ services:
    ports:
      - 8082:8082
      - 18082:18082
-    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1 -disk=ssd1'
+    command: 'volume -dataCenter dc2 -rack v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
    depends_on:
      - master0
      - master1
@@ -44,7 +56,7 @@ services:
    ports:
      - 8083:8083
      - 18083:18083
-    command: 'volume -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
+    command: 'volume -dataCenter dc3 -rack v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
    depends_on:
      - master0
      - master1
@@ -54,7 +66,8 @@ services:
    ports:
      - 8888:8888
      - 18888:18888
-    command: 'filer -master="master0:9333,master1:9334,master2:9335"'
+      - 8111:8111
+    command: 'filer -defaultReplicaPlacement 100 -iam -master="master0:9333,master1:9334,master2:9335"'
    depends_on:
      - master0
      - master1
@@ -65,11 +78,11 @@ services:
    image: chrislusf/seaweedfs:local
    ports:
      - 8333:8333
-    command: 's3 -filer="filer:8888"'
+    command: '-v=9 s3 -filer="filer:8888"'
    depends_on:
      - master0
      - master1
      - master2
      - volume1
      - volume2
-      - filer
+      - filer
@@ -38,7 +38,7 @@ services:
      S3TEST_CONF: "s3tests.conf"
      NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
      NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
-      NOSETESTS_EXCLUDE: "(bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
+      NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
    depends_on:
      - master
      - volume
@@ -24,7 +24,7 @@ case "$1" in
  'master')
  	ARGS="-mdir=/data -volumePreallocate -volumeSizeLimitMB=1024"
  	shift
-  	exec /usr/bin/weed master $ARGS $@
+  	exec /usr/bin/weed -logtostderr=true master $ARGS $@
 	;;

  'volume')
@@ -33,7 +33,7 @@ case "$1" in
  	  ARGS="-dir=/data"
  	fi
  	shift
-  	exec /usr/bin/weed volume $ARGS $@
+  	exec /usr/bin/weed -logtostderr=true volume $ARGS $@
 	;;

  'server')
@@ -42,31 +42,21 @@ case "$1" in
  	  ARGS="-dir=/data -master.volumePreallocate -master.volumeSizeLimitMB=1024"
  	fi
 	shift
-  	exec /usr/bin/weed server $ARGS $@
+  	exec /usr/bin/weed -logtostderr=true server $ARGS $@
  	;;

  'filer')
  	ARGS=""
  	shift
-  	exec /usr/bin/weed filer $ARGS $@
+  	exec /usr/bin/weed -logtostderr=true filer $ARGS $@
 	;;

  's3')
  	ARGS="-domainName=$S3_DOMAIN_NAME -key.file=$S3_KEY_FILE -cert.file=$S3_CERT_FILE"
  	shift
-  	exec /usr/bin/weed s3 $ARGS $@
+  	exec /usr/bin/weed -logtostderr=true s3 $ARGS $@
 	;;

-  'cronjob')
-	MASTER=${WEED_MASTER-localhost:9333}
-	FIX_REPLICATION_CRON_SCHEDULE=${CRON_SCHEDULE-*/7 * * * * *}
-	echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "lock; volume.fix.replication; unlock" | weed shell -master='$MASTER > /crontab
-	BALANCING_CRON_SCHEDULE=${CRON_SCHEDULE-25 * * * * *}
-	echo "$BALANCING_CRON_SCHEDULE" 'echo "lock; volume.balance -collection ALL_COLLECTIONS -force; unlock" | weed shell -master='$MASTER >> /crontab
-	echo "Running Crontab:"
-	cat /crontab
-	exec supercronic /crontab
-	;;
  *)
  	exec /usr/bin/weed $@
 	;;
@@ -0,0 +1,3 @@
+[rocksdb]
+enabled = true
+dir = "/data/filer_rocksdb"
@@ -28,16 +28,6 @@ services:
    depends_on:
      - master
      - volume
-  cronjob:
-    image: chrislusf/seaweedfs # use a remote image
-    command: 'cronjob'
-    environment:
-      # Run re-replication every 2 minutes
-      CRON_SCHEDULE: '*/2 * * * * *' # Default: '*/5 * * * * *'
-      WEED_MASTER: master:9333 # Default: localhost:9333
-    depends_on:
-      - master
-      - volume
  s3:
    image: chrislusf/seaweedfs # use a remote image
    ports:
@@ -8,8 +8,6 @@ require (
 	cloud.google.com/go/storage v1.16.1
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.14.0
-	github.com/BurntSushi/toml v0.3.1 // indirect
-	github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
 	github.com/OneOfOne/xxhash v1.2.2
 	github.com/Shopify/sarama v1.23.1
 	github.com/aws/aws-sdk-go v1.35.3
@@ -39,12 +37,13 @@ require (
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-redis/redis/v8 v8.4.4
+	github.com/go-redsync/redsync/v4 v4.4.1
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/go-zookeeper/zk v1.0.2 // indirect
 	github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
 	github.com/golang-jwt/jwt v3.2.1+incompatible
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.0.0
@@ -54,11 +53,10 @@ require (
 	github.com/googleapis/gax-go v2.0.2+incompatible // indirect
 	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
 	github.com/gorilla/mux v1.7.4
-	github.com/gorilla/websocket v1.4.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
-	github.com/hashicorp/go-multierror v1.0.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.0 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jcmturner/gofork v1.0.0 // indirect
@@ -68,12 +66,13 @@ require (
 	github.com/json-iterator/go v1.1.11
 	github.com/karlseguin/ccache/v2 v2.0.7
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.10.9 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/klauspost/cpuid v1.2.1 // indirect
 	github.com/klauspost/reedsolomon v1.9.2
 	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.0
+	github.com/linxGnu/grocksdb v1.6.38
 	github.com/magiconair/properties v1.8.1 // indirect
 	github.com/mailru/easyjson v0.7.1 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
@@ -100,7 +99,6 @@ require (
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
-	github.com/seaweedfs/fuse v1.2.0
 	github.com/seaweedfs/goexif v1.0.2
 	github.com/sirupsen/logrus v1.6.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
@@ -110,12 +108,11 @@ require (
 	github.com/spf13/viper v1.4.0
 	github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
 	github.com/stretchr/testify v1.7.0
+	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
-	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
-	github.com/tidwall/gjson v1.8.1
-	github.com/tidwall/match v1.0.3
-	github.com/tidwall/pretty v1.1.0 // indirect
-	github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210824090536-16d902a3c7e5
+	github.com/tidwall/gjson v1.10.2
+	github.com/tidwall/match v1.1.1
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
 	github.com/valyala/bytebufferpool v1.0.0
@@ -126,24 +123,24 @@ require (
 	github.com/xdg-go/scram v1.0.2 // indirect
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.etcd.io/etcd v3.3.25+incompatible
-	go.mongodb.org/mongo-driver v1.7.0
+	go.etcd.io/etcd/client/v3 v3.5.0
+	go.mongodb.org/mongo-driver v1.8.0
 	go.opencensus.io v0.23.0 // indirect
 	go.opentelemetry.io/otel v0.15.0 // indirect
 	gocloud.dev v0.20.0
 	gocloud.dev/pubsub/natspubsub v0.20.0
 	gocloud.dev/pubsub/rabbitpubsub v0.20.0
-	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect
+	golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
 	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf
-	golang.org/x/text v0.3.6 // indirect
-	golang.org/x/tools v0.1.5
+	golang.org/x/sys v0.0.0-20220111092808-5a964db01320
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/api v0.56.0
+	google.golang.org/api v0.57.0
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 // indirect
+	google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 // indirect
 	google.golang.org/grpc v1.40.0
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
@@ -165,38 +162,33 @@ require (
 )

 require (
-	github.com/coreos/etcd v3.3.10+incompatible // indirect
-	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
+	github.com/fluent/fluent-logger-golang v1.8.0
+	github.com/hanwen/go-fuse/v2 v2.1.0
+)
+
+require (
+	cloud.google.com/go/kms v1.0.0 // indirect
+	github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
-	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.2 // indirect
 	github.com/mattn/go-runewidth v0.0.7 // indirect
-	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 // indirect
-	github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd // indirect
-	github.com/pingcap/kvproto v0.0.0-20210806074406-317f69fb54b4 // indirect
-	github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 // indirect
-	github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307 // indirect
+	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
+	github.com/philhofer/fwd v1.1.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d // indirect
-	github.com/twmb/murmur3 v1.1.3 // indirect
+	github.com/tinylib/msgp v1.1.6 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.0 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
-	go.etcd.io/etcd/client/v3 v3.5.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.17.0 // indirect
-	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
 	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )

-// replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse
-// replace github.com/chrislusf/raft => /Users/chris/go/src/github.com/chrislusf/raft
+// replace github.com/chrislusf/raft => /Users/chrislu/go/src/github.com/chrislusf/raft
@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.67"
-version: "2.67"
+appVersion: "2.93"
+version: "2.93"
@@ -113,21 +113,6 @@ Inject extra environment vars in the format key:value, if populated
 {{- end -}}
 {{- end -}}

-{{/* Return the proper cronjob image */}}
-{{- define "cronjob.image" -}}
-{{- if .Values.cronjob.imageOverride -}}
-{{- $imageOverride := .Values.cronjob.imageOverride -}}
-{{- printf "%s" $imageOverride -}}
-{{- else -}}
-{{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}}
-{{- $repositoryName := .Values.image.repository | toString -}}
-{{- $name := .Values.global.imageName | toString -}}
-{{- $tag := .Chart.AppVersion | toString -}}
-{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}}
-{{- end -}}
-{{- end -}}
-
-
 {{/* check if any PVC exists */}}
 {{- define "volume.pvc_exists" -}}
 {{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
@@ -1,58 +0,0 @@
-{{- if .Values.cronjob }}
-{{- if .Values.cronjob.enabled }}
-apiVersion: batch/v1beta1
-kind: CronJob
-metadata:
-  name: {{ include "seaweedfs.fullname" . }}-cronjob
-spec:
-  schedule: "{{ .Values.cronjob.schedule }}"
-  startingDeadlineSeconds: 200
-  concurrencyPolicy: Forbid
-  failedJobsHistoryLimit: 2
-  successfulJobsHistoryLimit: 2
-  jobTemplate:
-    spec:
-      backoffLimit: 2
-      template:
-        spec:
-          {{- if .Values.cronjob.nodeSelector }}
-          nodeSelector:
-            {{ tpl .Values.cronjob.nodeSelector . | indent 12 | trim }}
-          {{- end }}
-          {{- if .Values.cronjob.tolerations }}
-          tolerations:
-            {{ tpl .Values.cronjob.tolerations . | nindent 12 | trim }}
-          {{- end }}
-          restartPolicy: OnFailure
-          containers:
-            - name: shell
-              image: {{ template "cronjob.image" . }}
-              imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }}
-              resources:
-              {{- toYaml .Values.cronjob.resources| nindent 16 }}
-              command:
-                - sh
-                - -c
-                - |
-                  set -ex
-                  echo -e "lock\n\
-                  volume.balance -force \
-                  {{ if .Values.volume.dataCenter }} -dataCenter {{ .Values.volume.dataCenter }}{{ end }}\
-                  {{ if .Values.cronjob.collection }} -collection {{ .Values.cronjob.collection }}{{ end }}\n\
-                  {{- if .Values.cronjob.enableFixReplication }}
-                  volume.fix.replication -collectionPattern={{ .Values.cronjob.collectionPattern }} \n\
-                  {{- end }}
-                  unlock\n" | \
-                  /usr/bin/weed shell \
-                  {{- if .Values.cronjob.master }}
-                  -master {{ .Values.cronjob.master }} \
-                  {{- else }}
-                  -master {{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}.svc:{{ .Values.master.port }} \
-                  {{- end }}
-                  {{- if .Values.cronjob.filer }}
-                  -filer {{ .Values.cronjob.filer }}
-                  {{- else }}
-                  -filer {{ template "seaweedfs.name" . }}-filer.{{ .Release.Namespace }}.svc:{{ .Values.filer.port }}
-                  {{- end }}
-{{- end }}
-{{- end }}
@@ -1,4 +1,5 @@
 {{- if .Values.filer.metricsPort }}
+{{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -15,4 +16,5 @@ spec:
  selector:
    app: {{ template "seaweedfs.name" . }}
    component: filer
+{{- end }}
 {{- end }}
@@ -133,11 +133,6 @@ spec:
              -encryptVolumeData \
              {{- end }}
              -ip=${POD_IP} \
-              {{- if .Values.filer.enable_peers }}
-              {{- if gt (.Values.filer.replicas | int) 1 }}
-              -peers=$(echo -n "{{ range $index := until (.Values.filer.replicas | int) }}${SEAWEEDFS_FULLNAME}-filer-{{ $index }}.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}{{ if lt $index (sub ($.Values.filer.replicas | int) 1) }},{{ end }}{{ end }}" | sed "s/$HOSTNAME.${SEAWEEDFS_FULLNAME}-filer:{{ $.Values.filer.port }}//" | sed 's/,$//; 's/^,//'; s/,,/,/;' )  \
-              {{- end }}
-              {{- end }}
              {{- if .Values.filer.s3.enabled }}
              -s3 \
              -s3.port={{ .Values.filer.s3.port }} \
@@ -154,6 +149,9 @@ spec:
              {{- if .Values.filer.s3.enableAuth }}
              -s3.config=/etc/sw/seaweedfs_s3_config \
              {{- end }}
+              {{- if .Values.filer.s3.auditLogConfig }}
+              -s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
+              {{- end }}
              {{- end }}
              -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
          {{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }}    
@@ -1,59 +1,67 @@
+{{- if .Values.filer.ingress.enabled }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1beta1
+{{- else }}
 apiVersion: extensions/v1beta1
+{{- end }}
 kind: Ingress
 metadata:
- name: ingress-{{ template "seaweedfs.name" . }}-filer
- annotations:
-   kubernetes.io/ingress.class: "nginx"
-   nginx.ingress.kubernetes.io/auth-type: "basic"
-   nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
-   nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Filer'
-   nginx.ingress.kubernetes.io/service-upstream: "true"
-   nginx.ingress.kubernetes.io/rewrite-target: /$1
-   nginx.ingress.kubernetes.io/use-regex: "true"
-   nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
-   nginx.ingress.kubernetes.io/ssl-redirect: "false"
-   nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
-   nginx.ingress.kubernetes.io/configuration-snippet: |
-     sub_filter '<head>' '<head> <base href="/sw-filer/">'; #add base url
-     sub_filter '="/' '="./';                               #make absolute paths to relative
-     sub_filter '=/' '=./';
-     sub_filter '/seaweedfsstatic' './seaweedfsstatic';
-     sub_filter_once off;
+  name: ingress-{{ template "seaweedfs.name" . }}-filer
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    {{ omit .Values.filer.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }}
 spec:
- rules:
- - http:
-     paths:
-     - path: /sw-filer/?(.*)
-       backend:
-         serviceName: {{ template "seaweedfs.name" . }}-filer
-         servicePort: {{ .Values.filer.port }}
+  ingressClassName: {{ .Values.filer.ingress.className | quote }}
+  rules:
+  - http:
+      paths:
+      - path: /sw-filer/?(.*)
+        pathType: ImplementationSpecific
+        backend:
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+          service:
+            name: {{ template "seaweedfs.name" . }}-filer
+            port:
+              number: {{ .Values.filer.port }}
+              #name:
+{{- else }}
+          serviceName: {{ template "seaweedfs.name" . }}-filer
+          servicePort: {{ .Values.filer.port }}
+{{- end }}
+{{- end }}
 ---
+{{- if .Values.master.ingress.enabled }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1beta1
+{{- else }}
 apiVersion: extensions/v1beta1
+{{- end }}
 kind: Ingress
 metadata:
  name: ingress-{{ template "seaweedfs.name" . }}-master
+  namespace: {{ .Release.Namespace }}
  annotations:
-    kubernetes.io/ingress.class: "nginx"
-    nginx.ingress.kubernetes.io/auth-type: "basic"
-    nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
-    nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master'
-    nginx.ingress.kubernetes.io/service-upstream: "true"
-    nginx.ingress.kubernetes.io/rewrite-target: /$1
-    nginx.ingress.kubernetes.io/use-regex: "true"
-    nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
-    nginx.ingress.kubernetes.io/ssl-redirect: "false"
-    nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
-    nginx.ingress.kubernetes.io/configuration-snippet: |
-      sub_filter '<head>' '<head> <base href="/sw-master/">'; #add base url
-      sub_filter '="/' '="./';                                #make absolute paths to relative
-      sub_filter '=/' '=./';
-      sub_filter '/seaweedfsstatic' './seaweedfsstatic';
-      sub_filter_once off;
+    {{ omit .Values.master.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }}
 spec:
+  ingressClassName: {{ .Values.master.ingress.className | quote }}
  rules:
  - http:
      paths:
      - path: /sw-master/?(.*)
+        pathType: ImplementationSpecific
        backend:
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+          service:
+            name: {{ template "seaweedfs.name" . }}-master
+            port:
+              number: {{ .Values.master.port }}
+              #name:
+{{- else }}
          serviceName: {{ template "seaweedfs.name" . }}-master
          servicePort: {{ .Values.master.port }}
+{{- end }}
+{{- end }}
@@ -93,6 +93,9 @@ spec:
              {{- if .Values.s3.enableAuth }}
              -config=/etc/sw/seaweedfs_s3_config \
              {{- end }}
+              {{- if .Values.s3.auditLogConfig }}
+              -auditLogConfig=/etc/sw/s3_auditLogConfig.json \
+              {{- end }}
              -filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }}
          volumeMounts:
            - name: logs
@@ -127,7 +130,7 @@ spec:
              name: swfs-s3
          readinessProbe:
            httpGet:
-              path: /
+              path: /status
              port: {{ .Values.s3.port }}
              scheme: HTTP
            initialDelaySeconds: 15
@@ -137,7 +140,7 @@ spec:
            timeoutSeconds: 10
          livenessProbe:
            httpGet:
-              path: /
+              path: /status
              port: {{ .Values.s3.port }}
              scheme: HTTP
            initialDelaySeconds: 20
@@ -1,4 +1,5 @@
 {{- if .Values.s3.metricsPort }}
+{{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -15,4 +16,5 @@ spec:
  selector:
    app: {{ template "seaweedfs.name" . }}
    component: s3
+{{- end }}
 {{- end }}
@@ -18,4 +18,12 @@ stringData:
  read_access_key_id: {{ $access_key_read }}
  read_secret_access_key: {{ $secret_key_read }}
  seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"{{ $access_key_admin }}","secretKey":"{{ $secret_key_admin }}"}],"actions":["Admin","Read","Write"]},{"name":"anvReadOnly","credentials":[{"accessKey":"{{ $access_key_read }}","secretKey":"{{ $secret_key_read }}"}],"actions":["Read"]}]}'
+  {{- if .Values.filer.s3.auditLogConfig }}
+  filer_s3_auditLogConfig.json: |
+    {{ toJson .Values.filer.s3.auditLogConfig | nindent 4 }}
+  {{- end }}
+  {{- if .Values.s3.auditLogConfig }}
+  s3_auditLogConfig.json: |
+    {{ toJson .Values.s3.auditLogConfig | nindent 4 }}
+  {{- end }}
 {{- end }}
@@ -1,7 +1,7 @@
 #hack for delete pod master after migration
 ---
 kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: seaweefds-rw-cr
 rules:
@@ -16,7 +16,7 @@ metadata:
  namespace: {{ .Release.Namespace }}
 ---
 kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: system:serviceaccount:seaweefds-rw-sa:default
 subjects:
@@ -26,4 +26,4 @@ subjects:
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
-  name: seaweefds-rw-cr
+  name: seaweefds-rw-cr
@@ -1,4 +1,5 @@
 {{- if .Values.volume.metricsPort }}
+{{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -15,4 +16,5 @@ spec:
  selector:
    app: {{ template "seaweedfs.name" . }}
    component: volume
+{{- end }}
 {{- end }}
@@ -40,7 +40,7 @@ spec:
      imagePullSecrets:
        - name: {{ .Values.global.imagePullSecrets }}
      {{- end }}
-      terminationGracePeriodSeconds: 10
+      terminationGracePeriodSeconds: 150
      {{- if .Values.volume.priorityClassName }}
      priorityClassName: {{ .Values.volume.priorityClassName | quote }}
      {{- end }}
@@ -109,6 +109,26 @@ master:
  # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
  priorityClassName: ""

+  ingress:
+    enabled: false
+    className: "nginx"
+    annotations:
+      nginx.ingress.kubernetes.io/auth-type: "basic"
+      nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
+      nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master'
+      nginx.ingress.kubernetes.io/service-upstream: "true"
+      nginx.ingress.kubernetes.io/rewrite-target: /$1
+      nginx.ingress.kubernetes.io/use-regex: "true"
+      nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
+      nginx.ingress.kubernetes.io/ssl-redirect: "false"
+      nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+      nginx.ingress.kubernetes.io/configuration-snippet: |
+        sub_filter '<head>' '<head> <base href="/sw-master/">'; #add base url
+        sub_filter '="/' '="./';                                #make absolute paths to relative
+        sub_filter '=/' '=./';
+        sub_filter '/seaweedfsstatic' './seaweedfsstatic';
+        sub_filter_once off;
+
  extraEnvironmentVars:
    WEED_MASTER_VOLUME_GROWTH_COPY_1: 7
    WEED_MASTER_VOLUME_GROWTH_COPY_2: 6
@@ -161,7 +181,7 @@ volume:

  # Directories to store data files. dir[,dir]... (default "/tmp")
  dir: "/data"
-  # Directories to store index files. dir[,dir]... (default "/tmp")
+  # Directories to store index files. dir[,dir]... (default is the same as "dir")
  dir_idx: null

  # Maximum numbers of volumes, count[,count]...
@@ -246,8 +266,6 @@ filer:
  maxMB: null
  # encrypt data on volume servers
  encryptVolumeData: false
-  # enable peers sync metadata, for leveldb (localdb for filer but with sync across)
-  enable_peers: false

  # Whether proxy or redirect to volume server during file GET request
  redirectOnRead: false
@@ -255,9 +273,6 @@ filer:
  # Limit sub dir listing size (default 100000)
  dirListLimit: 100000

-  # Turn off directory listing
-  disableDirListing: false
-
  # Disable http request, only gRpc operations are allowed
  disableHttp: false

@@ -311,6 +326,26 @@ filer:
  # ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
  priorityClassName: ""

+  ingress:
+    enabled: false
+    className: "nginx"
+    annotations:
+      nginx.ingress.kubernetes.io/auth-type: "basic"
+      nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
+      nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Filer'
+      nginx.ingress.kubernetes.io/service-upstream: "true"
+      nginx.ingress.kubernetes.io/rewrite-target: /$1
+      nginx.ingress.kubernetes.io/use-regex: "true"
+      nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
+      nginx.ingress.kubernetes.io/ssl-redirect: "false"
+      nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+      nginx.ingress.kubernetes.io/configuration-snippet: |
+        sub_filter '<head>' '<head> <base href="/sw-filer/">'; #add base url
+        sub_filter '="/' '="./';                               #make absolute paths to relative
+        sub_filter '=/' '=./';
+        sub_filter '/seaweedfsstatic' './seaweedfsstatic';
+        sub_filter_once off;
+
  # extraEnvVars is a list of extra enviroment variables to set with the stateful set.
  extraEnvironmentVars:
    WEED_MYSQL_ENABLED: "true"
@@ -340,6 +375,7 @@ filer:
    # enable user & permission to s3 (need to inject to all services)
    enableAuth: false
    skipAuthSecretCreation: false
+    auditLogConfig: {}

 s3:
  enabled: false
@@ -356,6 +392,7 @@ s3:
  # enable user & permission to s3 (need to inject to all services)
  enableAuth: false
  skipAuthSecretCreation: false
+  auditLogConfig: {}

  # Suffix of the host name, {bucket}.{domainName}
  domainName: ""
@@ -391,22 +428,6 @@ s3:
    size: ""
    storageClass: ""

-cronjob:
-  enabled: true
-  master: "seaweedfs-master:9333"
-  filer: "seaweedfs-filer-client:8888"
-  tolerations: ""
-  nodeSelector: |
-    sw-backend: "true"
-  replication:
-    enable: true
-    collectionPattern: ""
-  schedule: "*/7 * * * *"
-  resources: null
-  # balance all volumes among volume servers
-  # ALL|EACH_COLLECTION|<collection_name>
-  collection: ""
-

 certificates:
  commonName: "SeaweedFS CA"
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.7</version>
+    <version>2.85</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -14,7 +14,7 @@
    </parent>

    <properties>
-        <protobuf.version>3.9.1</protobuf.version>
+        <protobuf.version>3.16.1</protobuf.version>
        <!-- follow https://github.com/grpc/grpc-java -->
        <grpc.version>1.23.0</grpc.version>
        <guava.version>30.0-jre</guava.version>
@@ -60,7 +60,7 @@
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpmime</artifactId>
-            <version>4.5.6</version>
+            <version>4.5.13</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
@@ -135,7 +135,7 @@
            <plugin>
                <groupId>org.sonatype.plugins</groupId>
                <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.7</version>
+                <version>1.6.8</version>
                <extensions>true</extensions>
                <configuration>
                    <serverId>ossrh</serverId>
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.7</version>
+    <version>2.85</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -60,7 +60,7 @@
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpmime</artifactId>
-            <version>4.5.6</version>
+            <version>4.5.13</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
@@ -130,7 +130,7 @@
            <plugin>
                <groupId>org.sonatype.plugins</groupId>
                <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.7</version>
+                <version>1.6.8</version>
                <extensions>true</extensions>
                <configuration>
                    <serverId>ossrh</serverId>
@@ -5,7 +5,7 @@

    <groupId>com.github.chrislusf</groupId>
    <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.7</version>
+    <version>2.85</version>

    <parent>
        <groupId>org.sonatype.oss</groupId>
@@ -60,7 +60,7 @@
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpmime</artifactId>
-            <version>4.5.6</version>
+            <version>4.5.13</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
@@ -15,7 +15,7 @@ public class ChunkCache {
        }
        this.cache = CacheBuilder.newBuilder()
                .maximumSize(maxEntries)
-                .expireAfterAccess(1, TimeUnit.HOURS)
+                .expireAfterWrite(1, TimeUnit.HOURS)
                .build();
    }

@@ -14,7 +14,11 @@ public class FilerClient extends FilerGrpcClient {
    private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);

    public FilerClient(String host, int grpcPort) {
-        super(host, grpcPort);
+        super(host, grpcPort-10000, grpcPort);
+    }
+
+    public FilerClient(String host, int port, int grpcPort) {
+        super(host, port, grpcPort);
    }

    public static String toFileId(FilerProto.FileId fid) {
@@ -364,6 +368,7 @@ public class FilerClient extends FilerGrpcClient {
                .setPathPrefix(prefix)
                .setClientName(clientName)
                .setSinceNs(sinceNs)
+                .setClientId(this.randomClientId)
                .build()
        );
    }
@@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory;
 import javax.net.ssl.SSLException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Random;
 import java.util.concurrent.TimeUnit;

 public class FilerGrpcClient {
@@ -30,6 +31,7 @@ public class FilerGrpcClient {
    public final int VOLUME_SERVER_ACCESS_PUBLIC_URL = 1;
    public final int VOLUME_SERVER_ACCESS_FILER_PROXY = 2;
    public final Map<String, FilerProto.Locations> vidLocations = new HashMap<>();
+    protected int randomClientId;
    private final ManagedChannel channel;
    private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub;
    private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub;
@@ -40,11 +42,11 @@ public class FilerGrpcClient {
    private int volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT;
    private String filerAddress;

-    public FilerGrpcClient(String host, int grpcPort) {
-        this(host, grpcPort, sslContext);
+    public FilerGrpcClient(String host, int port, int grpcPort) {
+        this(host, port, grpcPort, sslContext);
    }

-    public FilerGrpcClient(String host, int grpcPort, SslContext sslContext) {
+    public FilerGrpcClient(String host, int port, int grpcPort, SslContext sslContext) {

        this(sslContext == null ?
                ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext()
@@ -54,7 +56,7 @@ public class FilerGrpcClient {
                        .negotiationType(NegotiationType.TLS)
                        .sslContext(sslContext));

-        filerAddress = String.format("%s:%d", host, grpcPort - 10000);
+        filerAddress = SeaweedUtil.joinHostPort(host, port);

        FilerProto.GetFilerConfigurationResponse filerConfigurationResponse =
                this.getBlockingStub().getFilerConfiguration(
@@ -62,6 +64,7 @@ public class FilerGrpcClient {
        cipher = filerConfigurationResponse.getCipher();
        collection = filerConfigurationResponse.getCollection();
        replication = filerConfigurationResponse.getReplication();
+        randomClientId = new Random().nextInt();

    }

@@ -0,0 +1,109 @@
+package seaweedfs.client;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+public class ReadChunks {
+
+    public static List<SeaweedRead.VisibleInterval> readResolvedChunks(List<FilerProto.FileChunk> chunkList) throws IOException {
+        List<Point> points = new ArrayList<>(chunkList.size() * 2);
+        for (FilerProto.FileChunk chunk : chunkList) {
+            points.add(new Point(chunk.getOffset(), chunk, true));
+            points.add(new Point(chunk.getOffset() + chunk.getSize(), chunk, false));
+        }
+        Collections.sort(points, new Comparator<Point>() {
+            @Override
+            public int compare(Point a, Point b) {
+                int x = (int) (a.x - b.x);
+                if (a.x != b.x) {
+                    return (int) (a.x - b.x);
+                }
+                if (a.ts != b.ts) {
+                    return (int) (a.ts - b.ts);
+                }
+                if (!a.isStart) {
+                    return -1;
+                }
+                return 1;
+            }
+        });
+
+        long prevX = 0;
+        List<SeaweedRead.VisibleInterval> visibles = new ArrayList<>();
+        ArrayList<Point> queue = new ArrayList<>();
+        for (Point point : points) {
+            if (point.isStart) {
+                if (queue.size() > 0) {
+                    int lastIndex = queue.size() - 1;
+                    Point lastPoint = queue.get(lastIndex);
+                    if (point.x != prevX && lastPoint.ts < point.ts) {
+                        addToVisibles(visibles, prevX, lastPoint, point);
+                        prevX = point.x;
+                    }
+                }
+                // insert into queue
+                for (int i = queue.size(); i >= 0; i--) {
+                    if (i == 0 || queue.get(i - 1).ts <= point.ts) {
+                        if (i == queue.size()) {
+                            prevX = point.x;
+                        }
+                        queue.add(i, point);
+                        break;
+                    }
+                }
+            } else {
+                int lastIndex = queue.size() - 1;
+                int index = lastIndex;
+                Point startPoint = null;
+                for (; index >= 0; index--) {
+                    startPoint = queue.get(index);
+                    if (startPoint.ts == point.ts) {
+                        queue.remove(index);
+                        break;
+                    }
+                }
+                if (index == lastIndex && startPoint != null) {
+                    addToVisibles(visibles, prevX, startPoint, point);
+                    prevX = point.x;
+                }
+            }
+        }
+
+        return visibles;
+
+    }
+
+    private static void addToVisibles(List<SeaweedRead.VisibleInterval> visibles, long prevX, Point startPoint, Point point) {
+        if (prevX < point.x) {
+            FilerProto.FileChunk chunk = startPoint.chunk;
+            visibles.add(new SeaweedRead.VisibleInterval(
+                    prevX,
+                    point.x,
+                    chunk.getFileId(),
+                    chunk.getMtime(),
+                    prevX - chunk.getOffset(),
+                    chunk.getOffset() == prevX && chunk.getSize() == prevX - startPoint.x,
+                    chunk.getCipherKey().toByteArray(),
+                    chunk.getIsCompressed()
+            ));
+        }
+    }
+
+    static class Point {
+        long x;
+        long ts;
+        FilerProto.FileChunk chunk;
+        boolean isStart;
+
+        public Point(long x, FilerProto.FileChunk chunk, boolean isStart) {
+            this.x = x;
+            this.ts = chunk.getMtime();
+            this.chunk = chunk;
+            this.isStart = isStart;
+        }
+    }
+
+}
@@ -14,10 +14,10 @@ public class RemoteUtil {
        String dir = SeaweedOutputStream.getParentDirectory(fullpath);
        String name = SeaweedOutputStream.getFileName(fullpath);

-        final FilerProto.DownloadToLocalResponse downloadToLocalResponse = filerClient.getBlockingStub()
-                .downloadToLocal(FilerProto.DownloadToLocalRequest.newBuilder()
+        final FilerProto.CacheRemoteObjectToLocalClusterResponse response = filerClient.getBlockingStub()
+                .cacheRemoteObjectToLocalCluster(FilerProto.CacheRemoteObjectToLocalClusterRequest.newBuilder()
                .setDirectory(dir).setName(name).build());

-        return downloadToLocalResponse.getEntry();
+        return response.getEntry();
    }
 }
@@ -226,96 +226,8 @@ public class SeaweedRead {

        chunkList = FileChunkManifest.resolveChunkManifest(filerClient, chunkList);

-        FilerProto.FileChunk[] chunks = chunkList.toArray(new FilerProto.FileChunk[0]);
-        Arrays.sort(chunks, new Comparator<FilerProto.FileChunk>() {
-            @Override
-            public int compare(FilerProto.FileChunk a, FilerProto.FileChunk b) {
-                // if just a.getMtime() - b.getMtime(), it will overflow!
-                if (a.getMtime() < b.getMtime()) {
-                    return -1;
-                } else if (a.getMtime() > b.getMtime()) {
-                    return 1;
-                }
-                return 0;
-            }
-        });
+        return ReadChunks.readResolvedChunks(chunkList);

-        List<VisibleInterval> visibles = new ArrayList<>();
-        for (FilerProto.FileChunk chunk : chunks) {
-            List<VisibleInterval> newVisibles = new ArrayList<>();
-            visibles = mergeIntoVisibles(visibles, newVisibles, chunk);
-        }
-
-        return visibles;
-    }
-
-    private static List<VisibleInterval> mergeIntoVisibles(List<VisibleInterval> visibles,
-                                                           List<VisibleInterval> newVisibles,
-                                                           FilerProto.FileChunk chunk) {
-        VisibleInterval newV = new VisibleInterval(
-                chunk.getOffset(),
-                chunk.getOffset() + chunk.getSize(),
-                chunk.getFileId(),
-                chunk.getMtime(),
-                0,
-                true,
-                chunk.getCipherKey().toByteArray(),
-                chunk.getIsCompressed()
-        );
-
-        // easy cases to speed up
-        if (visibles.size() == 0) {
-            visibles.add(newV);
-            return visibles;
-        }
-        if (visibles.get(visibles.size() - 1).stop <= chunk.getOffset()) {
-            visibles.add(newV);
-            return visibles;
-        }
-
-        for (VisibleInterval v : visibles) {
-            if (v.start < chunk.getOffset() && chunk.getOffset() < v.stop) {
-                newVisibles.add(new VisibleInterval(
-                        v.start,
-                        chunk.getOffset(),
-                        v.fileId,
-                        v.modifiedTime,
-                        v.chunkOffset,
-                        false,
-                        v.cipherKey,
-                        v.isCompressed
-                ));
-            }
-            long chunkStop = chunk.getOffset() + chunk.getSize();
-            if (v.start < chunkStop && chunkStop < v.stop) {
-                newVisibles.add(new VisibleInterval(
-                        chunkStop,
-                        v.stop,
-                        v.fileId,
-                        v.modifiedTime,
-                        v.chunkOffset + (chunkStop - v.start),
-                        false,
-                        v.cipherKey,
-                        v.isCompressed
-                ));
-            }
-            if (chunkStop <= v.start || v.stop <= chunk.getOffset()) {
-                newVisibles.add(v);
-            }
-        }
-        newVisibles.add(newV);
-
-        // keep everything sorted
-        for (int i = newVisibles.size() - 1; i >= 0; i--) {
-            if (i > 0 && newV.start < newVisibles.get(i - 1).start) {
-                newVisibles.set(i, newVisibles.get(i - 1));
-            } else {
-                newVisibles.set(i, newV);
-                break;
-            }
-        }
-
-        return newVisibles;
    }

    public static String parseVolumeId(String fileId) {
@@ -43,4 +43,14 @@ public class SeaweedUtil {
        String name = fullpath.substring(sep + 1);
        return new String[]{parent, name};
    }
+
+    public static String joinHostPort(String host, int port) {
+        if (host.startsWith("[") && host.endsWith("]")) {
+            return host + ":" + port;
+        }
+        if (host.indexOf(':')>=0) {
+            return "[" + host + "]:" + port;
+        }
+        return host + ":" + port;
+    }
 }
@@ -29,11 +29,31 @@ public class SeaweedWrite {
                                 final byte[] bytes,
                                 final long bytesOffset, final long bytesLength,
                                 final String path) throws IOException {
-        FilerProto.FileChunk.Builder chunkBuilder = writeChunk(
-                replication, filerClient, offset, bytes, bytesOffset, bytesLength, path);
-        synchronized (entry) {
-            entry.addChunks(chunkBuilder);
+
+        IOException lastException = null;
+        for (long waitTime = 1000L; waitTime < 10 * 1000; waitTime += waitTime / 2) {
+            try {
+                FilerProto.FileChunk.Builder chunkBuilder = writeChunk(
+                        replication, filerClient, offset, bytes, bytesOffset, bytesLength, path);
+                lastException = null;
+                synchronized (entry) {
+                    entry.addChunks(chunkBuilder);
+                }
+                break;
+            } catch (IOException ioe) {
+                LOG.debug("writeData:{}", ioe);
+                lastException = ioe;
+            }
+            try {
+                Thread.sleep(waitTime);
+            } catch (InterruptedException e) {
+            }
        }
+
+        if (lastException != null) {
+            throw lastException;
+        }
+
    }

    public static FilerProto.FileChunk.Builder writeChunk(final String replication,
@@ -59,7 +79,7 @@ public class SeaweedWrite {
        String fileId = response.getFileId();
        String auth = response.getAuth();

-        String targetUrl = filerClient.getChunkUrl(fileId, response.getUrl(), response.getPublicUrl());
+        String targetUrl = filerClient.getChunkUrl(fileId, response.getLocation().getUrl(), response.getLocation().getPublicUrl());

        ByteString cipherKeyString = com.google.protobuf.ByteString.EMPTY;
        byte[] cipherKey = null;
@@ -15,7 +15,7 @@ public class VolumeIdCache {
        }
        this.cache = CacheBuilder.newBuilder()
                .maximumSize(maxEntries)
-                .expireAfterAccess(5, TimeUnit.MINUTES)
+                .expireAfterWrite(5, TimeUnit.MINUTES)
                .build();
    }

@@ -30,6 +30,8 @@ service SeaweedFiler {

    rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
    }
+    rpc StreamRenameEntry (StreamRenameEntryRequest) returns (stream StreamRenameEntryResponse) {
+    }

    rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
    }
@@ -67,7 +69,7 @@ service SeaweedFiler {
    rpc KvPut (KvPutRequest) returns (KvPutResponse) {
    }

-    rpc DownloadToLocal (DownloadToLocalRequest) returns (DownloadToLocalResponse) {
+    rpc CacheRemoteObjectToLocalCluster (CacheRemoteObjectToLocalClusterRequest) returns (CacheRemoteObjectToLocalClusterResponse) {
    }
 }

@@ -112,6 +114,7 @@ message Entry {
    bytes content = 9; // if not empty, the file content

    RemoteEntry remote_entry = 10;
+    int64 quota = 11; // for bucket only. Positive/Negative means enabled/disabled.
 }

 message FullEntry {
@@ -168,6 +171,8 @@ message FuseAttributes {
    string symlink_target = 13;
    bytes md5 = 14;
    string disk_type = 15;
+    uint32 rdev = 16;
+    uint64 inode = 17;
 }

 message CreateEntryRequest {
@@ -225,6 +230,18 @@ message AtomicRenameEntryRequest {
 message AtomicRenameEntryResponse {
 }

+message StreamRenameEntryRequest {
+    string old_directory = 1;
+    string old_name = 2;
+    string new_directory = 3;
+    string new_name = 4;
+    repeated int32 signatures = 5;
+}
+message StreamRenameEntryResponse {
+    string directory = 1;
+    EventNotification event_notification = 2;
+    int64 ts_ns = 3;
+}
 message AssignVolumeRequest {
    int32 count = 1;
    string collection = 2;
@@ -233,18 +250,18 @@ message AssignVolumeRequest {
    string data_center = 5;
    string path = 6;
    string rack = 7;
+    string data_node = 9;
    string disk_type = 8;
 }

 message AssignVolumeResponse {
    string file_id = 1;
-    string url = 2;
-    string public_url = 3;
    int32 count = 4;
    string auth = 5;
    string collection = 6;
    string replication = 7;
    string error = 8;
+    Location location = 9;
 }

 message LookupVolumeRequest {
@@ -258,6 +275,7 @@ message Locations {
 message Location {
    string url = 1;
    string public_url = 2;
+    uint32 grpc_port = 3;
 }
 message LookupVolumeResponse {
    map<string, Locations> locations_map = 1;
@@ -314,6 +332,7 @@ message SubscribeMetadataRequest {
    int64 since_ns = 3;
    int32 signature = 4;
    repeated string path_prefixes = 6;
+    int32 client_id = 7;
 }
 message SubscribeMetadataResponse {
    string directory = 1;
@@ -382,6 +401,9 @@ message FilerConf {
        bool fsync = 6;
        uint32 volume_growth_count = 7;
        bool read_only = 8;
+        string data_center = 9;
+        string rack = 10;
+        string data_node = 11;
    }
    repeated PathConf locations = 2;
 }
@@ -389,10 +411,10 @@ message FilerConf {
 /////////////////////////
 // Remote Storage related
 /////////////////////////
-message DownloadToLocalRequest {
+message CacheRemoteObjectToLocalClusterRequest {
    string directory = 1;
    string name = 2;
 }
-message DownloadToLocalResponse {
+message CacheRemoteObjectToLocalClusterResponse {
    Entry entry = 1;
 }
@@ -6,6 +6,7 @@ import org.junit.Test;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Random;

 public class SeaweedReadTest {

@@ -13,17 +14,17 @@ public class SeaweedReadTest {
    public void testNonOverlappingVisibleIntervals() throws IOException {
        List<FilerProto.FileChunk> chunks = new ArrayList<>();
        chunks.add(FilerProto.FileChunk.newBuilder()
-            .setFileId("aaa")
-            .setOffset(0)
-            .setSize(100)
-            .setMtime(1000)
-            .build());
+                .setFileId("aaa")
+                .setOffset(0)
+                .setSize(100)
+                .setMtime(1000)
+                .build());
        chunks.add(FilerProto.FileChunk.newBuilder()
-            .setFileId("bbb")
-            .setOffset(100)
-            .setSize(133)
-            .setMtime(2000)
-            .build());
+                .setFileId("bbb")
+                .setOffset(100)
+                .setSize(133)
+                .setMtime(2000)
+                .build());

        List<SeaweedRead.VisibleInterval> visibleIntervals = SeaweedRead.nonOverlappingVisibleIntervals(null, chunks);
        for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) {
@@ -61,4 +62,106 @@ public class SeaweedReadTest {

    }

+
+    @Test
+    public void testReadResolvedChunks() throws IOException {
+        List<FilerProto.FileChunk> chunks = new ArrayList<>();
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("a")
+                .setOffset(0)
+                .setSize(100)
+                .setMtime(1)
+                .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("b")
+                .setOffset(50)
+                .setSize(100)
+                .setMtime(2)
+                .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("c")
+                .setOffset(200)
+                .setSize(50)
+                .setMtime(3)
+                .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("d")
+                .setOffset(250)
+                .setSize(50)
+                .setMtime(4)
+                .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+                .setFileId("e")
+                .setOffset(175)
+                .setSize(100)
+                .setMtime(5)
+                .build());
+
+        List<SeaweedRead.VisibleInterval> visibleIntervals = ReadChunks.readResolvedChunks(chunks);
+        for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) {
+            System.out.println("visible:" + visibleInterval);
+        }
+
+        Assert.assertEquals(4, visibleIntervals.size());
+
+        SeaweedRead.VisibleInterval visibleInterval = visibleIntervals.get(0);
+        Assert.assertEquals(visibleInterval.start, 0);
+        Assert.assertEquals(visibleInterval.stop, 50);
+        Assert.assertEquals(visibleInterval.modifiedTime, 1);
+        Assert.assertEquals(visibleInterval.fileId, "a");
+
+        visibleInterval = visibleIntervals.get(1);
+        Assert.assertEquals(visibleInterval.start, 50);
+        Assert.assertEquals(visibleInterval.stop, 150);
+        Assert.assertEquals(visibleInterval.modifiedTime, 2);
+        Assert.assertEquals(visibleInterval.fileId, "b");
+
+        visibleInterval = visibleIntervals.get(2);
+        Assert.assertEquals(visibleInterval.start, 175);
+        Assert.assertEquals(visibleInterval.stop, 275);
+        Assert.assertEquals(visibleInterval.modifiedTime, 5);
+        Assert.assertEquals(visibleInterval.fileId, "e");
+
+        visibleInterval = visibleIntervals.get(3);
+        Assert.assertEquals(visibleInterval.start, 275);
+        Assert.assertEquals(visibleInterval.stop, 300);
+        Assert.assertEquals(visibleInterval.modifiedTime, 4);
+        Assert.assertEquals(visibleInterval.fileId, "d");
+
+    }
+
+
+    @Test
+    public void testRandomizedReadResolvedChunks() throws IOException {
+        Random random = new Random();
+        int limit = 1024*1024;
+        long[] array = new long[limit];
+        List<FilerProto.FileChunk> chunks = new ArrayList<>();
+        for (long ts=0;ts<1024;ts++){
+            int x = random.nextInt(limit);
+            int y = random.nextInt(limit);
+            int size = Math.min(Math.abs(x-y), 1024);
+            chunks.add(randomWrite(array, Math.min(x,y), size, ts));
+        }
+
+        List<SeaweedRead.VisibleInterval> visibleIntervals = ReadChunks.readResolvedChunks(chunks);
+        for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) {
+            System.out.println("visible:" + visibleInterval);
+            for (int i = (int) visibleInterval.start; i<visibleInterval.stop; i++) {
+                Assert.assertEquals(array[i], visibleInterval.modifiedTime);
+            }
+        }
+
+    }
+    private FilerProto.FileChunk randomWrite(long[] array, int start, int size, long ts) {
+        for (int i=start;i<start+size;i++) {
+            array[i] = ts;
+        }
+        return FilerProto.FileChunk.newBuilder()
+                .setFileId("")
+                .setOffset(start)
+                .setSize(size)
+                .setMtime(ts)
+                .build();
+    }
 }
@@ -11,22 +11,22 @@
        <dependency>
            <groupId>com.github.chrislusf</groupId>
            <artifactId>seaweedfs-client</artifactId>
-            <version>1.6.7</version>
+            <version>2.85</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.github.chrislusf</groupId>
            <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>1.6.7</version>
+            <version>2.85</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
-            <version>2.9.2</version>
+            <version>2.10.1</version>
            <scope>compile</scope>
        </dependency>
    </dependencies>


-</project>
+</project>
@@ -36,7 +36,7 @@
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
-            <version>3.2.1</version>
+            <version>3.2.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
@@ -86,7 +86,7 @@
      <plugin>
        <groupId>org.sonatype.plugins</groupId>
        <artifactId>nexus-staging-maven-plugin</artifactId>
-        <version>1.6.7</version>
+        <version>1.6.8</version>
        <extensions>true</extensions>
        <configuration>
          <serverId>ossrh</serverId>
@@ -301,7 +301,7 @@
    </snapshotRepository>
  </distributionManagement>
  <properties>
-    <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
+    <seaweedfs.client.version>2.85</seaweedfs.client.version>
    <hadoop.version>2.9.2</hadoop.version>
  </properties>
 </project>
@@ -5,8 +5,8 @@
    <modelVersion>4.0.0</modelVersion>

    <properties>
-        <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
-        <hadoop.version>2.9.2</hadoop.version>
+        <seaweedfs.client.version>2.85</seaweedfs.client.version>
+        <hadoop.version>2.10.1</hadoop.version>
    </properties>

    <groupId>com.github.chrislusf</groupId>
@@ -105,7 +105,7 @@
            <plugin>
                <groupId>org.sonatype.plugins</groupId>
                <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.7</version>
+                <version>1.6.8</version>
                <extensions>true</extensions>
                <configuration>
                    <serverId>ossrh</serverId>
@@ -23,6 +23,7 @@ public class SeaweedFileSystem extends FileSystem {

    public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
    public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
+    public static final String FS_SEAWEED_FILER_PORT_GRPC = "fs.seaweed.filer.port.grpc";
    public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
    public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size";
    public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
@@ -50,9 +51,6 @@ public class SeaweedFileSystem extends FileSystem {
        // get host information from uri (overrides info in conf)
        String host = uri.getHost();
        host = (host == null) ? conf.get(FS_SEAWEED_FILER_HOST, "localhost") : host;
-        if (host == null) {
-            throw new IOException("Invalid host specified");
-        }
        conf.set(FS_SEAWEED_FILER_HOST, host);

        // get port information from uri, (overrides info in conf)
@@ -60,10 +58,12 @@ public class SeaweedFileSystem extends FileSystem {
        port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port;
        conf.setInt(FS_SEAWEED_FILER_PORT, port);

+        int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port+10000);
+
        setConf(conf);
        this.uri = uri;

-        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, conf);
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf);

    }

@@ -27,9 +27,8 @@ public class SeaweedFileSystemStore {
    private FilerClient filerClient;
    private Configuration conf;

-    public SeaweedFileSystemStore(String host, int port, Configuration conf) {
-        int grpcPort = 10000 + port;
-        filerClient = new FilerClient(host, grpcPort);
+    public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) {
+        filerClient = new FilerClient(host, port, grpcPort);
        this.conf = conf;
        String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
        if (volumeServerAccessMode.equals("publicUrl")) {
@@ -86,7 +86,7 @@
      <plugin>
        <groupId>org.sonatype.plugins</groupId>
        <artifactId>nexus-staging-maven-plugin</artifactId>
-        <version>1.6.7</version>
+        <version>1.6.8</version>
        <extensions>true</extensions>
        <configuration>
          <serverId>ossrh</serverId>
@@ -309,7 +309,7 @@
    </snapshotRepository>
  </distributionManagement>
  <properties>
-    <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
+    <seaweedfs.client.version>2.85</seaweedfs.client.version>
    <hadoop.version>3.1.1</hadoop.version>
  </properties>
 </project>
@@ -5,8 +5,8 @@
    <modelVersion>4.0.0</modelVersion>

    <properties>
-        <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
-        <hadoop.version>3.1.1</hadoop.version>
+        <seaweedfs.client.version>2.85</seaweedfs.client.version>
+        <hadoop.version>3.1.4</hadoop.version>
    </properties>

    <groupId>com.github.chrislusf</groupId>
@@ -105,7 +105,7 @@
            <plugin>
                <groupId>org.sonatype.plugins</groupId>
                <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.7</version>
+                <version>1.6.8</version>
                <extensions>true</extensions>
                <configuration>
                    <serverId>ossrh</serverId>
@@ -23,6 +23,7 @@ public class SeaweedFileSystem extends FileSystem {

    public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
    public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
+    public static final String FS_SEAWEED_FILER_PORT_GRPC = "fs.seaweed.filer.port.grpc";
    public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
    public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size";
    public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
@@ -50,9 +51,6 @@ public class SeaweedFileSystem extends FileSystem {
        // get host information from uri (overrides info in conf)
        String host = uri.getHost();
        host = (host == null) ? conf.get(FS_SEAWEED_FILER_HOST, "localhost") : host;
-        if (host == null) {
-            throw new IOException("Invalid host specified");
-        }
        conf.set(FS_SEAWEED_FILER_HOST, host);

        // get port information from uri, (overrides info in conf)
@@ -60,10 +58,12 @@ public class SeaweedFileSystem extends FileSystem {
        port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port;
        conf.setInt(FS_SEAWEED_FILER_PORT, port);

+        int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port+10000);
+
        setConf(conf);
        this.uri = uri;

-        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, conf);
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf);

    }

@@ -27,9 +27,8 @@ public class SeaweedFileSystemStore {
    private FilerClient filerClient;
    private Configuration conf;

-    public SeaweedFileSystemStore(String host, int port, Configuration conf) {
-        int grpcPort = 10000 + port;
-        filerClient = new FilerClient(host, grpcPort);
+    public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) {
+        filerClient = new FilerClient(host, port, grpcPort);
        this.conf = conf;
        String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
        if (volumeServerAccessMode.equals("publicUrl")) {
@@ -2,14 +2,15 @@ package basic

 import (
 	"fmt"
+	"io"
+	"os"
+	"strings"
+	"testing"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
-	"io/ioutil"
-	"os"
-	"strings"
-	"testing"
 )

 var (
@@ -108,8 +109,8 @@ func TestListBucket(t *testing.T) {
 func TestListObjectV2(t *testing.T) {

 	listObj, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
-		Bucket: aws.String(Bucket),
-		Prefix: aws.String("foo"),
+		Bucket:    aws.String(Bucket),
+		Prefix:    aws.String("foo"),
 		Delimiter: aws.String("/"),
 	})
 	if err != nil {
@@ -169,7 +170,7 @@ func TestObjectOp(t *testing.T) {
 		exitErrorf("Unable to get copy object, %v", err)
 	}

-	data, err := ioutil.ReadAll(getObj.Body)
+	data, err := io.ReadAll(getObj.Body)
 	if err != nil {
 		exitErrorf("Unable to read object data, %v", err)
 	}
@@ -0,0 +1,2 @@
+/s3-tests
+/tmp
@@ -0,0 +1,11 @@
+# the tests only support python 3.6, not newer
+FROM ubuntu:latest
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
+RUN git clone https://github.com/ceph/s3-tests.git
+WORKDIR s3-tests
+
+# we pin a certain commit
+RUN git checkout 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
+
+RUN ./bootstrap
@@ -0,0 +1,13 @@
+# Running S3 Compatibility tests against SeaweedFS
+
+This is using [the tests from CephFS](https://github.com/ceph/s3-tests).
+
+## Prerequisites
+
+- have Docker installed
+- this has been executed on Mac. On Linux, the hostname in `s3tests.conf`  needs to be adjusted.
+
+## Running tests
+
+- `./prepare.sh` to build the docker image
+- `./run.sh` to execute all tests
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -ex
+
+docker build  --progress=plain  -t s3tests .
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -ex
+
+killall -9 weed || echo "already stopped"
+rm -Rf tmp
+mkdir tmp
+docker stop s3test-instance || echo "already stopped"
+
+ulimit -n 10000
+../../../weed/weed server -filer -s3 -volume.max 0   -master.volumeSizeLimitMB 5 -dir "$(pwd)/tmp" 1>&2>weed.log &
+
+until $(curl --output /dev/null --silent --head --fail http://127.0.0.1:9333); do
+    printf '.'
+    sleep 5
+done
+sleep 3
+
+rm -Rf logs-full.txt logs-summary.txt
+# docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v `pwd`/s3tests.conf:/s3-tests/s3tests.conf -it s3tests    ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py:test_get_obj_tagging -v  -a 'resource=object,!bucket-policy,!versioning,!encryption'
+docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v `pwd`/s3tests.conf:/s3-tests/s3tests.conf -it s3tests    ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py -v  -a 'resource=object,!bucket-policy,!versioning,!encryption' | sed -n -e '/botocore.hooks/!p;//q' | tee logs-summary.txt
+
+docker stop s3test-instance || echo "already stopped"
+killall -9 weed
@@ -0,0 +1,109 @@
+[DEFAULT]
+## this section is just used for host, port and bucket_prefix
+
+# host set for rgw in vstart.sh
+host = host.docker.internal
+
+# port set for rgw in vstart.sh
+port = 8333
+
+## say "False" to disable TLS
+is_secure = False
+
+## say "False" to disable SSL Verify
+ssl_verify = False
+
+[fixtures]
+## all the buckets created will start with this prefix;
+## {random} will be filled with random characters to pad
+## the prefix to 30 characters long, and avoid collisions
+bucket prefix = yournamehere-{random}-
+
+[s3 main]
+# main display_name set in vstart.sh
+display_name = M. Tester
+
+# main user_idname set in vstart.sh
+user_id = testid
+
+# main email set in vstart.sh
+email = tester@ceph.com
+
+# zonegroup api_name for bucket location
+api_name = default
+
+## main AWS access key
+access_key = 0555b35654ad1656d804
+
+## main AWS secret key
+secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
+
+## replace with key id obtained when secret is created, or delete if KMS not tested
+#kms_keyid = 01234567-89ab-cdef-0123-456789abcdef
+
+[s3 alt]
+# alt display_name set in vstart.sh
+display_name = john.doe
+## alt email set in vstart.sh
+email = john.doe@example.com
+
+# alt user_id set in vstart.sh
+user_id = 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234
+
+# alt AWS access key set in vstart.sh
+access_key = NOPQRSTUVWXYZABCDEFG
+
+# alt AWS secret key set in vstart.sh
+secret_key = nopqrstuvwxyzabcdefghijklmnabcdefghijklm
+
+[s3 tenant]
+# tenant display_name set in vstart.sh
+display_name = testx$tenanteduser
+
+# tenant user_id set in vstart.sh
+user_id = 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+
+# tenant AWS secret key set in vstart.sh
+access_key = HIJKLMNOPQRSTUVWXYZA
+
+# tenant AWS secret key set in vstart.sh
+secret_key = opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab
+
+# tenant email set in vstart.sh
+email = tenanteduser@example.com
+
+#following section needs to be added for all sts-tests
+[iam]
+#used for iam operations in sts-tests
+#email from vstart.sh
+email = s3@example.com
+
+#user_id from vstart.sh
+user_id = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+
+#access_key from vstart.sh
+access_key = ABCDEFGHIJKLMNOPQRST
+
+#secret_key vstart.sh
+secret_key = abcdefghijklmnopqrstuvwxyzabcdefghijklmn
+
+#display_name from vstart.sh
+display_name = youruseridhere
+
+#following section needs to be added when you want to run Assume Role With Webidentity test
+[webidentity]
+#used for assume role with web identity test in sts-tests
+#all parameters will be obtained from ceph/qa/tasks/keycloak.py
+token=<access_token>
+
+aud=<obtained after introspecting token>
+
+sub=<obtained after introspecting token>
+
+azp=<obtained after introspecting token>
+
+user_token=<access token for a user, with attribute Department=[Engineering, Marketing>]
+
+thumbprint=<obtained from x509 certificate>
+
+KC_REALM=<name of the realm>
@@ -19,10 +19,10 @@ import (
 const (
 	maxPartSize        = int64(5 * 1024 * 1024)
 	maxRetries         = 3
-	awsAccessKeyID     = "Your access key"
-	awsSecretAccessKey = "Your secret key"
-	awsBucketRegion    = "S3 bucket region"
-	awsBucketName      = "newBucket"
+	awsAccessKeyID     = "any"
+	awsSecretAccessKey = "any"
+	awsBucketRegion    = "us‑west‑1"
+	awsBucketName      = "bucket1"
 )

 var (
@@ -37,7 +37,7 @@ func main() {
 	if err != nil {
 		fmt.Printf("bad credentials: %s", err)
 	}
-	cfg := aws.NewConfig().WithRegion(awsBucketRegion).WithCredentials(creds).WithDisableSSL(true).WithEndpoint("localhost:8333")
+	cfg := aws.NewConfig().WithRegion(awsBucketRegion).WithCredentials(creds).WithDisableSSL(true).WithEndpoint("localhost:8333").WithS3ForcePathStyle(true)
 	svc := s3.New(session.New(), cfg)

 	file, err := os.Open(*filename)
@@ -6,13 +6,9 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"io"
-	"math"
-	"os"
-	"strings"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/storage/idx"
@@ -20,6 +16,9 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"google.golang.org/grpc"
+	"io"
+	"math"
+	"os"
 )

 var (
@@ -45,13 +44,13 @@ func main() {
 	grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")

 	vid := uint32(*volumeId)
-	servers := strings.Split(*serversStr, ",")
+	servers := pb.ServerAddresses(*serversStr).ToAddresses()
 	if len(servers) < 2 {
 		glog.Fatalf("You must specify more than 1 server\n")
 	}
-	var referenceServer string
+	var referenceServer pb.ServerAddress
 	var maxOffset int64
-	allFiles := map[string]map[types.NeedleId]needleState{}
+	allFiles := map[pb.ServerAddress]map[types.NeedleId]needleState{}
 	for _, addr := range servers {
 		files, offset, err := getVolumeFiles(vid, addr)
 		if err != nil {
@@ -121,9 +120,9 @@ type needleState struct {
 	size  types.Size
 }

-func getVolumeFiles(v uint32, addr string) (map[types.NeedleId]needleState, int64, error) {
+func getVolumeFiles(v uint32, addr pb.ServerAddress) (map[types.NeedleId]needleState, int64, error) {
 	var idxFile *bytes.Reader
-	err := operation.WithVolumeServerClient(addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
+	err := operation.WithVolumeServerClient(false, addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 		copyFileClient, err := vs.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
@@ -179,9 +178,9 @@ func getVolumeFiles(v uint32, addr string) (map[types.NeedleId]needleState, int6
 	return files, maxOffset, nil
 }

-func getNeedleFileId(v uint32, nid types.NeedleId, addr string) (string, error) {
+func getNeedleFileId(v uint32, nid types.NeedleId, addr pb.ServerAddress) (string, error) {
 	var id string
-	err := operation.WithVolumeServerClient(addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
+	err := operation.WithVolumeServerClient(false, addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
 		resp, err := vs.VolumeNeedleStatus(context.Background(), &volume_server_pb.VolumeNeedleStatusRequest{
 			VolumeId: v,
 			NeedleId: uint64(nid),
@@ -12,10 +12,11 @@ import (
 )

 var (
-	dir       = flag.String("dir", "/tmp", "directory to create files")
-	n         = flag.Int("n", 100, "the number of metadata")
-	tailFiler = flag.String("filer", "localhost:8888", "the filer address")
-	isWrite   = flag.Bool("write", false, "only write")
+	dir           = flag.String("dir", "/tmp", "directory to create files")
+	n             = flag.Int("n", 100, "the number of metadata")
+	tailFiler     = flag.String("filer", "localhost:8888", "the filer address")
+	isWrite       = flag.Bool("write", false, "only write")
+	writeInterval = flag.Duration("writeInterval", 0, "write interval, e.g., 1s")
 )

 func main() {
@@ -50,10 +51,11 @@ func main() {
 }

 func startGenerateMetadata() {
-	pb.WithFilerClient(*tailFiler, grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error {
+	pb.WithFilerClient(false, pb.ServerAddress(*tailFiler), grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error {

 		for i := 0; i < *n; i++ {
 			name := fmt.Sprintf("file%d", i)
+			glog.V(0).Infof("write %s/%s", *dir, name)
 			if err := filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{
 				Directory: *dir,
 				Entry: &filer_pb.Entry{
@@ -63,6 +65,9 @@ func startGenerateMetadata() {
 				fmt.Printf("create entry %s: %v\n", name, err)
 				return err
 			}
+			if *writeInterval > 0 {
+				time.Sleep(*writeInterval)
+			}
 		}

 		return nil
@@ -72,7 +77,7 @@ func startGenerateMetadata() {

 func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {

-	tailErr := pb.FollowMetadata(*tailFiler, grpc.WithInsecure(), "tail", *dir, nil, 0, 0, eachEntryFunc, false)
+	tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithInsecure(), "tail", 0, *dir, nil, 0, 0, eachEntryFunc, false)

 	if tailErr != nil {
 		fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"log"
 	"math/rand"
 	"time"
@@ -32,7 +33,7 @@ func main() {
 	go func() {
 		for {
 			println("vacuum threshold", *garbageThreshold)
-			_, _, err := util.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=%f", *master, *garbageThreshold))
+			_, _, err := util.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=%f", pb.ServerAddress(*master).ToHttpAddress(), *garbageThreshold))
 			if err != nil {
 				log.Fatalf("vacuum: %v", err)
 			}
@@ -52,7 +53,7 @@ func main() {
 }

 func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, string) {
-	assignResult, err := operation.Assign(func() string { return *master }, grpcDialOption, &operation.VolumeAssignRequest{
+	assignResult, err := operation.Assign(func() pb.ServerAddress { return pb.ServerAddress(*master) }, grpcDialOption, &operation.VolumeAssignRequest{
 		Count:       1,
 		Replication: *replication,
 	})
@@ -0,0 +1,63 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"io"
+)
+
+var (
+	volumeServer   = flag.String("volumeServer", "localhost:8080", "a volume server")
+	volumeId       = flag.Int("volumeId", -1, "a volume id to stream read")
+	grpcDialOption grpc.DialOption
+)
+
+func main() {
+	flag.Parse()
+
+	util.LoadConfiguration("security", false)
+	grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	vid := uint32(*volumeId)
+
+	eachNeedleFunc := func(resp *volume_server_pb.ReadAllNeedlesResponse) error {
+		fmt.Printf("%d,%x%08x %d\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob))
+		return nil
+	}
+
+	err := operation.WithVolumeServerClient(true, pb.ServerAddress(*volumeServer), grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		copyFileClient, err := vs.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{
+			VolumeIds: []uint32{vid},
+		})
+		if err != nil {
+			return err
+		}
+		for {
+			resp, err := copyFileClient.Recv()
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			if err != nil {
+				return err
+			}
+			if err = eachNeedleFunc(resp); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		fmt.Printf("read %s: %v\n", *volumeServer, err)
+	}
+
+}
@@ -5,7 +5,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"mime/multipart"
@@ -45,7 +44,7 @@ func main() {
 			defer wg.Done()

 			client := &http.Client{Transport: &http.Transport{
-				MaxIdleConns:     1024,
+				MaxIdleConns:        1024,
 				MaxIdleConnsPerHost: 1024,
 			}}
 			r := rand.New(rand.NewSource(time.Now().UnixNano() + int64(x)))
@@ -128,7 +127,7 @@ func uploadFileToFiler(client *http.Client, data []byte, filename, destination s
 		if err != nil {
 			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
 		}
-		io.Copy(ioutil.Discard, resp.Body)
+		io.Copy(io.Discard, resp.Body)
 		resp.Body.Close()
 	}

@@ -5,7 +5,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"mime/multipart"
@@ -36,7 +35,7 @@ func main() {

 	var fileNames []string

-	files, err := ioutil.ReadDir(*dir)
+	files, err := os.ReadDir(*dir)
 	if err != nil {
 		log.Fatalf("fail to read dir %v: %v", *dir, err)
 	}
@@ -142,7 +141,7 @@ func uploadFileToFiler(client *http.Client, filename, destination string) (size
 		if err != nil {
 			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
 		}
-		io.Copy(ioutil.Discard, resp.Body)
+		io.Copy(io.Discard, resp.Body)
 		resp.Body.Close()
 	}

@@ -2,6 +2,7 @@ package main

 import (
 	"flag"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"log"
 	"time"

@@ -37,7 +38,7 @@ func main() {
 		sinceTimeNs = time.Now().Add(-*rewindDuration).UnixNano()
 	}

-	err := operation.TailVolume(func()string{return *master}, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
+	err := operation.TailVolume(func()pb.ServerAddress{return pb.ServerAddress(*master)}, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
 		if n.Size == 0 {
 			println("-", n.String())
 			return nil
@@ -2,10 +2,13 @@ BINARY = weed

 SOURCE_DIR = .

-all: debug_mount
+all: install

 .PHONY : clean debug_mount

+install:
+	go install
+
 clean:
 	go clean $(SOURCE_DIR)
 	rm -f $(BINARY)
@@ -16,15 +19,15 @@ debug_shell:

 debug_mount:
 	go build -gcflags="all=-N -l"
-	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/ -umask=000

 debug_server:
 	go build -gcflags="all=-N -l"
-	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=/Volumes/mobile_disk/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1

 debug_volume:
 	go build -gcflags="all=-N -l"
-	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- volume -dir=/Volumes/mobile_disk/100 -port 8564 -max=30 -preStopSeconds=2
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- volume -dir=~/tmp/100 -port 8564 -max=30 -preStopSeconds=2

 debug_webdav:
 	go build -gcflags="all=-N -l"
@@ -49,4 +52,3 @@ debug_filer_remote_sync_buckets:
 debug_master_follower:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 master.follower
-
@@ -0,0 +1,264 @@
+package cluster
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"math"
+	"sync"
+	"time"
+)
+
+const (
+	MasterType = "master"
+	FilerType  = "filer"
+	BrokerType = "broker"
+)
+
+type ClusterNode struct {
+	Address   pb.ServerAddress
+	Version   string
+	counter   int
+	createdTs time.Time
+}
+
+type Leaders struct {
+	leaders [3]pb.ServerAddress
+}
+
+type Cluster struct {
+	filers       map[pb.ServerAddress]*ClusterNode
+	filersLock   sync.RWMutex
+	filerLeaders *Leaders
+	brokers      map[pb.ServerAddress]*ClusterNode
+	brokersLock  sync.RWMutex
+}
+
+func NewCluster() *Cluster {
+	return &Cluster{
+		filers:       make(map[pb.ServerAddress]*ClusterNode),
+		filerLeaders: &Leaders{},
+		brokers:      make(map[pb.ServerAddress]*ClusterNode),
+	}
+}
+
+func (cluster *Cluster) AddClusterNode(nodeType string, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
+	switch nodeType {
+	case FilerType:
+		cluster.filersLock.Lock()
+		defer cluster.filersLock.Unlock()
+		if existingNode, found := cluster.filers[address]; found {
+			existingNode.counter++
+			return nil
+		}
+		cluster.filers[address] = &ClusterNode{
+			Address:   address,
+			Version:   version,
+			counter:   1,
+			createdTs: time.Now(),
+		}
+		return cluster.ensureFilerLeaders(true, nodeType, address)
+	case BrokerType:
+		cluster.brokersLock.Lock()
+		defer cluster.brokersLock.Unlock()
+		if existingNode, found := cluster.brokers[address]; found {
+			existingNode.counter++
+			return nil
+		}
+		cluster.brokers[address] = &ClusterNode{
+			Address:   address,
+			Version:   version,
+			counter:   1,
+			createdTs: time.Now(),
+		}
+		return []*master_pb.KeepConnectedResponse{
+			{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsAdd:    true,
+				},
+			},
+		}
+	case MasterType:
+	}
+	return nil
+}
+
+func (cluster *Cluster) RemoveClusterNode(nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
+	switch nodeType {
+	case FilerType:
+		cluster.filersLock.Lock()
+		defer cluster.filersLock.Unlock()
+		if existingNode, found := cluster.filers[address]; !found {
+			return nil
+		} else {
+			existingNode.counter--
+			if existingNode.counter <= 0 {
+				delete(cluster.filers, address)
+				return cluster.ensureFilerLeaders(false, nodeType, address)
+			}
+		}
+	case BrokerType:
+		cluster.brokersLock.Lock()
+		defer cluster.brokersLock.Unlock()
+		if existingNode, found := cluster.brokers[address]; !found {
+			return nil
+		} else {
+			existingNode.counter--
+			if existingNode.counter <= 0 {
+				delete(cluster.brokers, address)
+				return []*master_pb.KeepConnectedResponse{
+					{
+						ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+							NodeType: nodeType,
+							Address:  string(address),
+							IsAdd:    false,
+						},
+					},
+				}
+			}
+		}
+	case MasterType:
+	}
+	return nil
+}
+
+func (cluster *Cluster) ListClusterNode(nodeType string) (nodes []*ClusterNode) {
+	switch nodeType {
+	case FilerType:
+		cluster.filersLock.RLock()
+		defer cluster.filersLock.RUnlock()
+		for _, node := range cluster.filers {
+			nodes = append(nodes, node)
+		}
+	case BrokerType:
+		cluster.brokersLock.RLock()
+		defer cluster.brokersLock.RUnlock()
+		for _, node := range cluster.brokers {
+			nodes = append(nodes, node)
+		}
+	case MasterType:
+	}
+	return
+}
+
+func (cluster *Cluster) IsOneLeader(address pb.ServerAddress) bool {
+	return cluster.filerLeaders.isOneLeader(address)
+}
+
+func (cluster *Cluster) ensureFilerLeaders(isAdd bool, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
+	if isAdd {
+		if cluster.filerLeaders.addLeaderIfVacant(address) {
+			// has added the address as one leader
+			result = append(result, &master_pb.KeepConnectedResponse{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsLeader: true,
+					IsAdd:    true,
+				},
+			})
+		} else {
+			result = append(result, &master_pb.KeepConnectedResponse{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsLeader: false,
+					IsAdd:    true,
+				},
+			})
+		}
+	} else {
+		if cluster.filerLeaders.removeLeaderIfExists(address) {
+
+			result = append(result, &master_pb.KeepConnectedResponse{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsLeader: true,
+					IsAdd:    false,
+				},
+			})
+
+			// pick the freshest one, since it is less likely to go away
+			var shortestDuration int64 = math.MaxInt64
+			now := time.Now()
+			var candidateAddress pb.ServerAddress
+			for _, node := range cluster.filers {
+				if cluster.filerLeaders.isOneLeader(node.Address) {
+					continue
+				}
+				duration := now.Sub(node.createdTs).Nanoseconds()
+				if duration < shortestDuration {
+					shortestDuration = duration
+					candidateAddress = node.Address
+				}
+			}
+			if candidateAddress != "" {
+				cluster.filerLeaders.addLeaderIfVacant(candidateAddress)
+				// added a new leader
+				result = append(result, &master_pb.KeepConnectedResponse{
+					ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+						NodeType: nodeType,
+						Address:  string(candidateAddress),
+						IsLeader: true,
+						IsAdd:    true,
+					},
+				})
+			}
+		} else {
+			result = append(result, &master_pb.KeepConnectedResponse{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsLeader: false,
+					IsAdd:    false,
+				},
+			})
+		}
+	}
+	return
+}
+
+func (leaders *Leaders) addLeaderIfVacant(address pb.ServerAddress) (hasChanged bool) {
+	if leaders.isOneLeader(address) {
+		return
+	}
+	for i := 0; i < len(leaders.leaders); i++ {
+		if leaders.leaders[i] == "" {
+			leaders.leaders[i] = address
+			hasChanged = true
+			return
+		}
+	}
+	return
+}
+func (leaders *Leaders) removeLeaderIfExists(address pb.ServerAddress) (hasChanged bool) {
+	if !leaders.isOneLeader(address) {
+		return
+	}
+	for i := 0; i < len(leaders.leaders); i++ {
+		if leaders.leaders[i] == address {
+			leaders.leaders[i] = ""
+			hasChanged = true
+			return
+		}
+	}
+	return
+}
+func (leaders *Leaders) isOneLeader(address pb.ServerAddress) bool {
+	for i := 0; i < len(leaders.leaders); i++ {
+		if leaders.leaders[i] == address {
+			return true
+		}
+	}
+	return false
+}
+func (leaders *Leaders) GetLeaders() (addresses []pb.ServerAddress) {
+	for i := 0; i < len(leaders.leaders); i++ {
+		if leaders.leaders[i] != "" {
+			addresses = append(addresses, leaders.leaders[i])
+		}
+	}
+	return
+}
@@ -0,0 +1,47 @@
+package cluster
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestClusterAddRemoveNodes(t *testing.T) {
+	c := NewCluster()
+
+	c.AddClusterNode("filer", pb.ServerAddress("111:1"), "23.45")
+	c.AddClusterNode("filer", pb.ServerAddress("111:2"), "23.45")
+	assert.Equal(t, []pb.ServerAddress{
+		pb.ServerAddress("111:1"),
+		pb.ServerAddress("111:2"),
+	}, c.filerLeaders.GetLeaders())
+
+	c.AddClusterNode("filer", pb.ServerAddress("111:3"), "23.45")
+	c.AddClusterNode("filer", pb.ServerAddress("111:4"), "23.45")
+	assert.Equal(t, []pb.ServerAddress{
+		pb.ServerAddress("111:1"),
+		pb.ServerAddress("111:2"),
+		pb.ServerAddress("111:3"),
+	}, c.filerLeaders.GetLeaders())
+
+	c.AddClusterNode("filer", pb.ServerAddress("111:5"), "23.45")
+	c.AddClusterNode("filer", pb.ServerAddress("111:6"), "23.45")
+	c.RemoveClusterNode("filer", pb.ServerAddress("111:4"))
+	assert.Equal(t, []pb.ServerAddress{
+		pb.ServerAddress("111:1"),
+		pb.ServerAddress("111:2"),
+		pb.ServerAddress("111:3"),
+	}, c.filerLeaders.GetLeaders())
+
+	// remove oldest
+	c.RemoveClusterNode("filer", pb.ServerAddress("111:1"))
+	assert.Equal(t, []pb.ServerAddress{
+		pb.ServerAddress("111:6"),
+		pb.ServerAddress("111:2"),
+		pb.ServerAddress("111:3"),
+	}, c.filerLeaders.GetLeaders())
+
+	// remove oldest
+	c.RemoveClusterNode("filer", pb.ServerAddress("111:1"))
+
+}
@@ -2,6 +2,7 @@ package command

 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"

 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
@@ -72,12 +73,12 @@ func runBackup(cmd *Command, args []string) bool {
 	vid := needle.VolumeId(*s.volumeId)

 	// find volume location, replication, ttl info
-	lookup, err := operation.LookupVolumeId(func() string { return *s.master }, grpcDialOption, vid.String())
+	lookup, err := operation.LookupVolumeId(func() pb.ServerAddress { return pb.ServerAddress(*s.master) }, grpcDialOption, vid.String())
 	if err != nil {
 		fmt.Printf("Error looking up volume %d: %v\n", vid, err)
 		return true
 	}
-	volumeServer := lookup.Locations[0].Url
+	volumeServer := lookup.Locations[0].ServerAddress()

 	stats, err := operation.GetVolumeSyncStatus(volumeServer, grpcDialOption, uint32(vid))
 	if err != nil {
@@ -119,7 +120,7 @@ func runBackup(cmd *Command, args []string) bool {
 	}

 	if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
-		if err = v.Compact2(30*1024*1024*1024, 0); err != nil {
+		if err = v.Compact2(30*1024*1024*1024, 0, nil); err != nil {
 			fmt.Printf("Compact Volume before synchronizing %v\n", err)
 			return true
 		}
@@ -3,6 +3,7 @@ package command
 import (
 	"bufio"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"io"
 	"math"
 	"math/rand"
@@ -10,7 +11,6 @@ import (
 	"runtime"
 	"runtime/pprof"
 	"sort"
-	"strings"
 	"sync"
 	"time"

@@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool {
 		defer pprof.StopCPUProfile()
 	}

-	b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", 0, "", strings.Split(*b.masters, ","))
+	b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddresses())
 	go b.masterClient.KeepConnectedToMaster()
 	b.masterClient.WaitUntilConnected()

@@ -21,6 +21,7 @@ var Commands = []*Command{
 	cmdFilerCopy,
 	cmdFilerMetaBackup,
 	cmdFilerMetaTail,
+	cmdFilerRemoteGateway,
 	cmdFilerRemoteSynchronize,
 	cmdFilerReplicate,
 	cmdFilerSynchronize,
@@ -50,7 +50,7 @@ func runCompact(cmd *Command, args []string) bool {
 			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	} else {
-		if err = v.Compact2(preallocate, 0); err != nil {
+		if err = v.Compact2(preallocate, 0, nil); err != nil {
 			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	}
@@ -2,16 +2,17 @@ package command

 import (
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"google.golang.org/grpc"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
 	"strings"

+	"google.golang.org/grpc"
+
 	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )

@@ -49,7 +50,7 @@ func runDownload(cmd *Command, args []string) bool {
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")

 	for _, fid := range args {
-		if e := downloadToFile(func() string { return *d.server }, grpcDialOption, fid, util.ResolvePath(*d.dir)); e != nil {
+		if e := downloadToFile(func() pb.ServerAddress { return pb.ServerAddress(*d.server) }, grpcDialOption, fid, util.ResolvePath(*d.dir)); e != nil {
 			fmt.Println("Download Error: ", fid, e)
 		}
 	}
@@ -81,7 +82,7 @@ func downloadToFile(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpti
 	}
 	defer f.Close()
 	if isFileList {
-		content, err := ioutil.ReadAll(rc.Body)
+		content, err := io.ReadAll(rc.Body)
 		if err != nil {
 			return err
 		}
@@ -118,7 +119,7 @@ func fetchContent(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption
 		return "", nil, e
 	}
 	defer util.CloseResponse(rc)
-	content, e = ioutil.ReadAll(rc.Body)
+	content, e = io.ReadAll(rc.Body)
 	return
 }

@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"net/http"
 	"os"
-	"strconv"
-	"strings"
 	"time"

 	"google.golang.org/grpc/reflection"
@@ -30,10 +28,12 @@ var (
 )

 type FilerOptions struct {
-	masters                 *string
+	masters                 []pb.ServerAddress
+	mastersString           *string
 	ip                      *string
 	bindIp                  *string
 	port                    *int
+	portGrpc                *int
 	publicPort              *int
 	collection              *string
 	defaultReplicaPlacement *string
@@ -45,7 +45,6 @@ type FilerOptions struct {
 	enableNotification      *bool
 	disableHttp             *bool
 	cipher                  *bool
-	peers                   *string
 	metricsHttpPort         *int
 	saveToFilerLimit        *int
 	defaultLevelDbDirectory *string
@@ -56,11 +55,12 @@ type FilerOptions struct {

 func init() {
 	cmdFiler.Run = runFiler // break init cycle
-	f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
+	f.mastersString = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
 	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
 	f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
 	f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to")
 	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
+	f.portGrpc = cmdFiler.Flag.Int("port.grpc", 0, "filer server grpc listen port")
 	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
 	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
@@ -70,7 +70,6 @@ func init() {
 	f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
 	f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
 	f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
-	f.peers = cmdFiler.Flag.String("peers", "", "all filers sharing the same filer store in comma separated ip:port list")
 	f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
 	f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
@@ -85,7 +84,8 @@ func init() {
 	filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
 	filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
 	filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
-	filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", false, "allow empty folders")
+	filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
+	filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")

 	// start webdav on filer
 	filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
@@ -96,7 +96,7 @@ func init() {
 	filerWebDavOptions.tlsPrivateKey = cmdFiler.Flag.String("webdav.key.file", "", "path to the TLS private key file")
 	filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
 	filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
-	filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB")
+	filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")

 	// start iam on filer
 	filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
@@ -134,10 +134,11 @@ func runFiler(cmd *Command, args []string) bool {

 	go stats_collect.StartMetricsServer(*f.metricsHttpPort)

-	filerAddress := fmt.Sprintf("%s:%d", *f.ip, *f.port)
+	filerAddress := util.JoinHostPort(*f.ip, *f.port)
 	startDelay := time.Duration(2)
 	if *filerStartS3 {
 		filerS3Options.filer = &filerAddress
+		filerS3Options.bindIp = f.bindIp
 		go func() {
 			time.Sleep(startDelay * time.Second)
 			filerS3Options.startS3Server()
@@ -156,13 +157,15 @@ func runFiler(cmd *Command, args []string) bool {

 	if *filerStartIam {
 		filerIamOptions.filer = &filerAddress
-		filerIamOptions.masters = f.masters
+		filerIamOptions.masters = f.mastersString
 		go func() {
 			time.Sleep(startDelay * time.Second)
 			filerIamOptions.startIamServer()
 		}()
 	}

+	f.masters = pb.ServerAddresses(*f.mastersString).ToAddresses()
+
 	f.startFiler()

 	return true
@@ -176,16 +179,16 @@ func (fo *FilerOptions) startFiler() {
 	if *fo.publicPort != 0 {
 		publicVolumeMux = http.NewServeMux()
 	}
+	if *fo.portGrpc == 0 {
+		*fo.portGrpc = 10000 + *fo.port
+	}

 	defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")

-	var peers []string
-	if *fo.peers != "" {
-		peers = strings.Split(*fo.peers, ",")
-	}
+	filerAddress := pb.NewServerAddress(*fo.ip, *fo.port, *fo.portGrpc)

 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
-		Masters:               strings.Split(*fo.masters, ","),
+		Masters:               fo.masters,
 		Collection:            *fo.collection,
 		DefaultReplication:    *fo.defaultReplicaPlacement,
 		DisableDirListing:     *fo.disableDirListing,
@@ -195,11 +198,9 @@ func (fo *FilerOptions) startFiler() {
 		Rack:                  *fo.rack,
 		DefaultLevelDbDir:     defaultLevelDbDirectory,
 		DisableHttp:           *fo.disableHttp,
-		Host:                  *fo.ip,
-		Port:                  uint32(*fo.port),
+		Host:                  filerAddress,
 		Cipher:                *fo.cipher,
 		SaveToFilerLimit:      int64(*fo.saveToFilerLimit),
-		Filers:                peers,
 		ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
 	})
 	if nfs_err != nil {
@@ -207,7 +208,7 @@ func (fo *FilerOptions) startFiler() {
 	}

 	if *fo.publicPort != 0 {
-		publicListeningAddress := *fo.bindIp + ":" + strconv.Itoa(*fo.publicPort)
+		publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
 		glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
 		publicListener, e := util.NewListener(publicListeningAddress, 0)
 		if e != nil {
@@ -222,7 +223,7 @@ func (fo *FilerOptions) startFiler() {

 	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port)
 	filerListener, e := util.NewListener(
-		*fo.bindIp+":"+strconv.Itoa(*fo.port),
+		util.JoinHostPort(*fo.bindIp, *fo.port),
 		time.Duration(10)*time.Second,
 	)
 	if e != nil {
@@ -230,8 +231,8 @@ func (fo *FilerOptions) startFiler() {
 	}

 	// starting grpc server
-	grpcPort := *fo.port + 10000
-	grpcL, err := util.NewListener(*fo.bindIp+":"+strconv.Itoa(grpcPort), 0)
+	grpcPort := *fo.portGrpc
+	grpcL, err := util.NewListener(util.JoinHostPort(*fo.bindIp, grpcPort), 0)
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
 	}
@@ -54,8 +54,10 @@ func runFilerBackup(cmd *Command, args []string) bool {

 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")

+	clientId := util.RandomInt32()
+
 	for {
-		err := doFilerBackup(grpcDialOption, &filerBackupOptions)
+		err := doFilerBackup(grpcDialOption, &filerBackupOptions, clientId)
 		if err != nil {
 			glog.Errorf("backup from %s: %v", *filerBackupOptions.filer, err)
 			time.Sleep(1747 * time.Millisecond)
@@ -69,7 +71,7 @@ const (
 	BackupKeyPrefix = "backup."
 )

-func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions) error {
+func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions, clientId int32) error {

 	// find data sink
 	config := util.GetViper()
@@ -78,7 +80,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 		return fmt.Errorf("no data sink configured in replication.toml")
 	}

-	sourceFiler := *backupOption.filer
+	sourceFiler := pb.ServerAddress(*backupOption.filer)
 	sourcePath := *backupOption.path
 	timeAgo := *backupOption.timeAgo
 	targetPath := dataSink.GetSinkToDirectory()
@@ -102,7 +104,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti

 	// create filer sink
 	filerSource := &source.FilerSource{}
-	filerSource.DoInitialize(sourceFiler, pb.ServerToGrpcAddress(sourceFiler), sourcePath, *backupOption.proxyByFiler)
+	filerSource.DoInitialize(sourceFiler.ToHttpAddress(), sourceFiler.ToGrpcAddress(), sourcePath, *backupOption.proxyByFiler)
 	dataSink.SetSourceFiler(filerSource)

 	processEventFn := genProcessFunction(sourcePath, targetPath, dataSink, debug)
@@ -112,7 +114,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 		return setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), lastTsNs)
 	})

-	return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(),
+	return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), clientId,
 		sourcePath, nil, startFrom.UnixNano(), 0, processEventFnWithOffset, false)

 }
@@ -8,7 +8,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 	"google.golang.org/grpc"
-	"math"
 	"net/url"
 	"os"
 	"strings"
@@ -23,7 +22,7 @@ var (

 type FilerCatOptions struct {
 	grpcDialOption grpc.DialOption
-	filerAddress   string
+	filerAddress   pb.ServerAddress
 	filerClient    filer_pb.SeaweedFilerClient
 	output         *string
 }
@@ -78,7 +77,7 @@ func runFilerCat(cmd *Command, args []string) bool {
 		return false
 	}

-	filerCat.filerAddress = filerUrl.Host
+	filerCat.filerAddress = pb.ServerAddress(filerUrl.Host)
 	filerCat.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")

 	dir, name := util.FullPath(urlPath).DirAndName()
@@ -97,7 +96,7 @@ func runFilerCat(cmd *Command, args []string) bool {
 		writer = f
 	}

-	pb.WithFilerClient(filerCat.filerAddress, filerCat.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	pb.WithFilerClient(false, filerCat.filerAddress, filerCat.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Name:      name,
@@ -115,7 +114,7 @@ func runFilerCat(cmd *Command, args []string) bool {

 		filerCat.filerClient = client

-		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, int64(filer.FileSize(respLookupEntry.Entry)))

 	})

@@ -3,11 +3,8 @@ package command
 import (
 	"context"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/filer"
 	"io"
-	"io/ioutil"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -17,14 +14,14 @@ import (

 	"google.golang.org/grpc"

-	"github.com/chrislusf/seaweedfs/weed/util/grace"
-
+	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/util/grace"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )

@@ -92,35 +89,21 @@ func runCopy(cmd *Command, args []string) bool {
 	filerDestination := args[len(args)-1]
 	fileOrDirs := args[0 : len(args)-1]

-	filerUrl, err := url.Parse(filerDestination)
+	filerAddress, urlPath, err := pb.ParseUrl(filerDestination)
 	if err != nil {
 		fmt.Printf("The last argument should be a URL on filer: %v\n", err)
 		return false
 	}
-	urlPath := filerUrl.Path
 	if !strings.HasSuffix(urlPath, "/") {
 		fmt.Printf("The last argument should be a folder and end with \"/\"\n")
 		return false
 	}

-	if filerUrl.Port() == "" {
-		fmt.Printf("The filer port should be specified.\n")
-		return false
-	}
-
-	filerPort, parseErr := strconv.ParseUint(filerUrl.Port(), 10, 64)
-	if parseErr != nil {
-		fmt.Printf("The filer port parse error: %v\n", parseErr)
-		return false
-	}
-
-	filerGrpcPort := filerPort + 10000
-	filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort)
 	copy.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")

-	masters, collection, replication, dirBuckets, maxMB, cipher, err := readFilerConfiguration(copy.grpcDialOption, filerGrpcAddress)
+	masters, collection, replication, dirBuckets, maxMB, cipher, err := readFilerConfiguration(copy.grpcDialOption, filerAddress)
 	if err != nil {
-		fmt.Printf("read from filer %s: %v\n", filerGrpcAddress, err)
+		fmt.Printf("read from filer %s: %v\n", filerAddress, err)
 		return false
 	}
 	if strings.HasPrefix(urlPath, dirBuckets+"/") {
@@ -174,9 +157,8 @@ func runCopy(cmd *Command, args []string) bool {
 		go func() {
 			defer waitGroup.Done()
 			worker := FileCopyWorker{
-				options:          &copy,
-				filerHost:        filerUrl.Host,
-				filerGrpcAddress: filerGrpcAddress,
+				options:      &copy,
+				filerAddress: filerAddress,
 			}
 			if err := worker.copyFiles(fileCopyTaskChan); err != nil {
 				fmt.Fprintf(os.Stderr, "copy file error: %v\n", err)
@@ -189,8 +171,8 @@ func runCopy(cmd *Command, args []string) bool {
 	return true
 }

-func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress string) (masters []string, collection, replication string, dirBuckets string, maxMB uint32, cipher bool, err error) {
-	err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress pb.ServerAddress) (masters []string, collection, replication string, dirBuckets string, maxMB uint32, cipher bool, err error) {
+	err = pb.WithGrpcFilerClient(false, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 		if err != nil {
 			return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
@@ -228,9 +210,9 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
 	}

 	if mode.IsDir() {
-		files, _ := ioutil.ReadDir(fileOrDir)
+		files, _ := os.ReadDir(fileOrDir)
 		for _, subFileOrDir := range files {
-			cleanedDestDirectory := filepath.Clean(destPath + fi.Name())
+			cleanedDestDirectory := destPath + fi.Name()
 			if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), cleanedDestDirectory+"/", fileCopyTaskChan); err != nil {
 				return err
 			}
@@ -241,9 +223,8 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
 }

 type FileCopyWorker struct {
-	options          *CopyOptions
-	filerHost        string
-	filerGrpcAddress string
+	options      *CopyOptions
+	filerAddress pb.ServerAddress
 }

 func (worker *FileCopyWorker) copyFiles(fileCopyTaskChan chan FileCopyTask) error {
@@ -321,7 +302,7 @@ func (worker *FileCopyWorker) checkExistingFileFirst(task FileCopyTask, f *os.Fi
 		return
 	}

-	err = pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Directory: task.destinationUrlPath,
@@ -356,14 +337,14 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 	if task.fileMode&os.ModeDir == 0 && task.fileSize > 0 {

 		mimeType = detectMimeType(f)
-		data, err := ioutil.ReadAll(f)
+		data, err := io.ReadAll(f)
 		if err != nil {
 			return err
 		}

-		// assign a volume
-		err = util.Retry("assignVolume", func() error {
-			return pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err = util.Retry("upload", func() error {
+			// assign a volume
+			assignErr := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

 				request := &filer_pb.AssignVolumeRequest{
 					Count:       1,
@@ -381,43 +362,49 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 				if assignResult.Error != "" {
 					return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error)
 				}
-				if assignResult.Url == "" {
+				if assignResult.Location.Url == "" {
 					return fmt.Errorf("assign volume failure %v: %v", request, assignResult)
 				}
 				return nil
 			})
+			if assignErr != nil {
+				return assignErr
+			}
+
+			// upload data
+			targetUrl := "http://" + assignResult.Location.Url + "/" + assignResult.FileId
+			uploadOption := &operation.UploadOption{
+				UploadUrl:         targetUrl,
+				Filename:          fileName,
+				Cipher:            worker.options.cipher,
+				IsInputCompressed: false,
+				MimeType:          mimeType,
+				PairMap:           nil,
+				Jwt:               security.EncodedJwt(assignResult.Auth),
+			}
+			uploadResult, err := operation.UploadData(data, uploadOption)
+			if err != nil {
+				return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
+			}
+			if uploadResult.Error != "" {
+				return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
+			}
+			if *worker.options.verbose {
+				fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
+			}
+
+			fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName)
+			chunks = append(chunks, uploadResult.ToPbFileChunk(assignResult.FileId, 0))
+
+			return nil
 		})
 		if err != nil {
-			return fmt.Errorf("Failed to assign from %v: %v\n", worker.options.masters, err)
+			return fmt.Errorf("upload %v: %v\n", fileName, err)
 		}

-		targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId
-		uploadOption := &operation.UploadOption{
-			UploadUrl:         targetUrl,
-			Filename:          fileName,
-			Cipher:            worker.options.cipher,
-			IsInputCompressed: false,
-			MimeType:          mimeType,
-			PairMap:           nil,
-			Jwt:               security.EncodedJwt(assignResult.Auth),
-		}
-		uploadResult, err := operation.UploadData(data, uploadOption)
-		if err != nil {
-			return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
-		}
-		if uploadResult.Error != "" {
-			return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
-		}
-		if *worker.options.verbose {
-			fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
-		}
-
-		chunks = append(chunks, uploadResult.ToPbFileChunk(assignResult.FileId, 0))
-
-		fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerHost, task.destinationUrlPath, fileName)
 	}

-	if err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
@@ -443,7 +430,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 		}
 		return nil
 	}); err != nil {
-		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
+		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName, err)
 	}

 	return nil
@@ -474,7 +461,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 			var assignResult *filer_pb.AssignVolumeResponse
 			var assignError error
 			err := util.Retry("assignVolume", func() error {
-				return pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+				return pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 					request := &filer_pb.AssignVolumeRequest{
 						Count:       1,
 						Replication: *worker.options.replication,
@@ -495,10 +482,11 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 				})
 			})
 			if err != nil {
-				fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err)
+				uploadError = fmt.Errorf("Failed to assign from %v: %v\n", worker.options.masters, err)
+				return
 			}

-			targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId
+			targetUrl := "http://" + assignResult.Location.Url + "/" + assignResult.FileId
 			if collection == "" {
 				collection = assignResult.Collection
 			}
@@ -508,7 +496,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,

 			uploadOption := &operation.UploadOption{
 				UploadUrl:         targetUrl,
-				Filename:          fileName+"-"+strconv.FormatInt(i+1, 10),
+				Filename:          fileName + "-" + strconv.FormatInt(i+1, 10),
 				Cipher:            worker.options.cipher,
 				IsInputCompressed: false,
 				MimeType:          "",
@@ -542,8 +530,8 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 		for _, chunk := range chunks {
 			fileIds = append(fileIds, chunk.FileId)
 		}
-		operation.DeleteFiles(func() string {
-			return copy.masters[0]
+		operation.DeleteFiles(func() pb.ServerAddress {
+			return pb.ServerAddress(copy.masters[0])
 		}, false, worker.options.grpcDialOption, fileIds)
 		return uploadError
 	}
@@ -553,7 +541,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 		return fmt.Errorf("create manifest: %v", manifestErr)
 	}

-	if err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
@@ -579,10 +567,10 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 		}
 		return nil
 	}); err != nil {
-		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
+		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName, err)
 	}

-	fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerHost, task.destinationUrlPath, fileName)
+	fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName)

 	return nil
 }
@@ -611,7 +599,7 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off
 	var fileId, host string
 	var auth security.EncodedJwt

-	if flushErr := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if flushErr := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {

 		ctx := context.Background()

@@ -633,7 +621,7 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off
 				return fmt.Errorf("assign volume failure %v: %v", request, resp.Error)
 			}

-			fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth)
+			fileId, host, auth = resp.FileId, resp.Location.Url, security.EncodedJwt(resp.Auth)
 			collection, replication = resp.Collection, resp.Replication

 			return nil
@@ -27,7 +27,8 @@ type FilerMetaBackupOptions struct {
 	restart           *bool
 	backupFilerConfig *string

-	store filer.FilerStore
+	store    filer.FilerStore
+	clientId int32
 }

 func init() {
@@ -36,6 +37,7 @@ func init() {
 	metaBackup.filerDirectory = cmdFilerMetaBackup.Flag.String("filerDir", "/", "a folder on the filer")
 	metaBackup.restart = cmdFilerMetaBackup.Flag.Bool("restart", false, "copy the full metadata before async incremental backup")
 	metaBackup.backupFilerConfig = cmdFilerMetaBackup.Flag.String("config", "", "path to filer.toml specifying backup filer store")
+	metaBackup.clientId = util.RandomInt32()
 }

 var cmdFilerMetaBackup = &Command{
@@ -160,24 +162,21 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
 		ctx := context.Background()
 		message := resp.EventNotification

-		if message.OldEntry == nil && message.NewEntry == nil {
+		if filer_pb.IsEmpty(resp) {
 			return nil
-		}
-		if message.OldEntry == nil && message.NewEntry != nil {
+		} else if filer_pb.IsCreate(resp) {
 			println("+", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
 			entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry)
 			return store.InsertEntry(ctx, entry)
-		}
-		if message.OldEntry != nil && message.NewEntry == nil {
+		} else if filer_pb.IsDelete(resp) {
 			println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name))
 			return store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name))
-		}
-		if message.OldEntry != nil && message.NewEntry != nil {
-			if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
-				println("~", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
-				entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry)
-				return store.UpdateEntry(ctx, entry)
-			}
+		} else if filer_pb.IsUpdate(resp) {
+			println("~", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
+			entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry)
+			return store.UpdateEntry(ctx, entry)
+		} else {
+			// renaming
 			println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name))
 			if err := store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name)); err != nil {
 				return err
@@ -195,7 +194,7 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
 		return metaBackup.setOffset(lastTime)
 	})

-	return pb.FollowMetadata(*metaBackup.filerAddress, metaBackup.grpcDialOption, "meta_backup",
+	return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, "meta_backup", metaBackup.clientId,
 		*metaBackup.filerDirectory, nil, startTime.UnixNano(), 0, processEventFnWithOffset, false)

 }
@@ -222,9 +221,9 @@ func (metaBackup *FilerMetaBackupOptions) setOffset(lastWriteTime time.Time) err

 var _ = filer_pb.FilerClient(&FilerMetaBackupOptions{})

-func (metaBackup *FilerMetaBackupOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
+func (metaBackup *FilerMetaBackupOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {

-	return pb.WithFilerClient(*metaBackup.filerAddress, metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithFilerClient(streamingMode, pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		return fn(client)
 	})

@@ -6,7 +6,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/golang/protobuf/jsonpb"
 	jsoniter "github.com/json-iterator/go"
-	"github.com/olivere/elastic/v7"
+	elastic "github.com/olivere/elastic/v7"
 	"os"
 	"path/filepath"
 	"strings"
@@ -30,6 +30,8 @@ var cmdFilerMetaTail = &Command{
 	weed filer.meta.tail -timeAgo=30h | jq .
 	weed filer.meta.tail -timeAgo=30h | jq .eventNotification.newEntry.name

+	weed filer.meta.tail -timeAgo=30h -es=http://<elasticSearchServerHost>:<port> -es.index=seaweedfs
+
  `,
 }

@@ -46,6 +48,7 @@ func runFilerMetaTail(cmd *Command, args []string) bool {

 	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+	clientId := util.RandomInt32()

 	var filterFunc func(dir, fname string) bool
 	if *tailPattern != "" {
@@ -71,12 +74,12 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
 	}

 	shouldPrint := func(resp *filer_pb.SubscribeMetadataResponse) bool {
+		if filer_pb.IsEmpty(resp) {
+			return false
+		}
 		if filterFunc == nil {
 			return true
 		}
-		if resp.EventNotification.OldEntry == nil && resp.EventNotification.NewEntry == nil {
-			return false
-		}
 		if resp.EventNotification.OldEntry != nil && filterFunc(resp.Directory, resp.EventNotification.OldEntry.Name) {
 			return true
 		}
@@ -103,7 +106,7 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
 		}
 	}

-	tailErr := pb.FollowMetadata(*tailFiler, grpcDialOption, "tail",
+	tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, "tail", clientId,
 		*tailTarget, nil, time.Now().Add(-*tailStart).UnixNano(), 0,
 		func(resp *filer_pb.SubscribeMetadataResponse) error {
 			if !shouldPrint(resp) {
@@ -0,0 +1,119 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/chrislusf/seaweedfs/weed/replication/source"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"os"
+	"time"
+)
+
+type RemoteGatewayOptions struct {
+	filerAddress             *string
+	grpcDialOption           grpc.DialOption
+	readChunkFromFiler       *bool
+	timeAgo                  *time.Duration
+	createBucketAt           *string
+	createBucketRandomSuffix *bool
+	include                  *string
+	exclude                  *string
+
+	mappings    *remote_pb.RemoteStorageMapping
+	remoteConfs map[string]*remote_pb.RemoteConf
+	bucketsDir  string
+	clientId    int32
+}
+
+var _ = filer_pb.FilerClient(&RemoteGatewayOptions{})
+
+func (option *RemoteGatewayOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	return pb.WithFilerClient(streamingMode, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		return fn(client)
+	})
+}
+func (option *RemoteGatewayOptions) AdjustedUrl(location *filer_pb.Location) string {
+	return location.Url
+}
+
+var (
+	remoteGatewayOptions RemoteGatewayOptions
+)
+
+func init() {
+	cmdFilerRemoteGateway.Run = runFilerRemoteGateway // break init cycle
+	remoteGatewayOptions.filerAddress = cmdFilerRemoteGateway.Flag.String("filer", "localhost:8888", "filer of the SeaweedFS cluster")
+	remoteGatewayOptions.createBucketAt = cmdFilerRemoteGateway.Flag.String("createBucketAt", "", "one remote storage name to create new buckets in")
+	remoteGatewayOptions.createBucketRandomSuffix = cmdFilerRemoteGateway.Flag.Bool("createBucketWithRandomSuffix", true, "add randomized suffix to bucket name to avoid conflicts")
+	remoteGatewayOptions.readChunkFromFiler = cmdFilerRemoteGateway.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers")
+	remoteGatewayOptions.timeAgo = cmdFilerRemoteGateway.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
+	remoteGatewayOptions.include = cmdFilerRemoteGateway.Flag.String("include", "", "pattens of new bucket names, e.g., s3*")
+	remoteGatewayOptions.exclude = cmdFilerRemoteGateway.Flag.String("exclude", "", "pattens of new bucket names, e.g., local*")
+	remoteGatewayOptions.clientId = util.RandomInt32()
+}
+
+var cmdFilerRemoteGateway = &Command{
+	UsageLine: "filer.remote.gateway",
+	Short:     "resumable continuously write back bucket creation, deletion, and other local updates to remote object store",
+	Long: `resumable continuously write back bucket creation, deletion, and other local updates to remote object store
+
+	filer.remote.gateway listens on filer local buckets update events. 
+	If any bucket is created, deleted, or updated, it will mirror the changes to remote object store.
+
+		weed filer.remote.sync -createBucketAt=cloud1
+
+`,
+}
+
+func runFilerRemoteGateway(cmd *Command, args []string) bool {
+
+	util.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+	remoteGatewayOptions.grpcDialOption = grpcDialOption
+
+	filerAddress := pb.ServerAddress(*remoteGatewayOptions.filerAddress)
+
+	filerSource := &source.FilerSource{}
+	filerSource.DoInitialize(
+		filerAddress.ToHttpAddress(),
+		filerAddress.ToGrpcAddress(),
+		"/", // does not matter
+		*remoteGatewayOptions.readChunkFromFiler,
+	)
+
+	remoteGatewayOptions.bucketsDir = "/buckets"
+	// check buckets again
+	remoteGatewayOptions.WithFilerClient(false, func(filerClient filer_pb.SeaweedFilerClient) error {
+		resp, err := filerClient.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		remoteGatewayOptions.bucketsDir = resp.DirBuckets
+		return nil
+	})
+
+	// read filer remote storage mount mappings
+	if detectErr := remoteGatewayOptions.collectRemoteStorageConf(); detectErr != nil {
+		fmt.Fprintf(os.Stderr, "read mount info: %v\n", detectErr)
+		return true
+	}
+
+	// synchronize /buckets folder
+	fmt.Printf("synchronize buckets in %s ...\n", remoteGatewayOptions.bucketsDir)
+	util.RetryForever("filer.remote.sync buckets", func() error {
+		return remoteGatewayOptions.followBucketUpdatesAndUploadToRemote(filerSource)
+	}, func(err error) bool {
+		if err != nil {
+			glog.Errorf("synchronize %s: %v", remoteGatewayOptions.bucketsDir, err)
+		}
+		return true
+	})
+	return true
+
+}
@@ -13,11 +13,12 @@ import (
 	"github.com/golang/protobuf/proto"
 	"math"
 	"math/rand"
+	"path/filepath"
 	"strings"
 	"time"
 )

-func (option *RemoteSyncOptions) followBucketUpdatesAndUploadToRemote(filerSource *source.FilerSource) error {
+func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSource *source.FilerSource) error {

 	// read filer remote storage mount mappings
 	if detectErr := option.collectRemoteStorageConf(); detectErr != nil {
@@ -32,16 +33,16 @@ func (option *RemoteSyncOptions) followBucketUpdatesAndUploadToRemote(filerSourc
 	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
 		lastTime := time.Unix(0, lastTsNs)
 		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
-		return remote_storage.SetSyncOffset(option.grpcDialOption, *option.filerAddress, option.bucketsDir, lastTsNs)
+		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, lastTsNs)
 	})

-	lastOffsetTs := collectLastSyncOffset(option, option.bucketsDir)
+	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, *option.timeAgo)

-	return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, "filer.remote.sync",
+	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId,
 		option.bucketsDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
 }

-func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
+func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {

 	handleCreateBucket := func(entry *filer_pb.Entry) error {
 		if !entry.IsDirectory {
@@ -75,12 +76,36 @@ func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.
 		}

 		bucketName := strings.ToLower(entry.Name)
-		if *option.createBucketRandomSuffix {
-			// https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
-			if len(bucketName)+5 > 63 {
-				bucketName = bucketName[:58]
+		if *option.include != "" {
+			if ok, _ := filepath.Match(*option.include, entry.Name); !ok {
+				return nil
 			}
-			bucketName = fmt.Sprintf("%s-%4d", bucketName, rand.Uint32()%10000)
+		}
+		if *option.exclude != "" {
+			if ok, _ := filepath.Match(*option.exclude, entry.Name); ok {
+				return nil
+			}
+		}
+
+		bucketPath := util.FullPath(option.bucketsDir).Child(entry.Name)
+		remoteLocation, found := option.mappings.Mappings[string(bucketPath)]
+		if !found {
+			if *option.createBucketRandomSuffix {
+				// https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
+				if len(bucketName)+5 > 63 {
+					bucketName = bucketName[:58]
+				}
+				bucketName = fmt.Sprintf("%s-%04d", bucketName, rand.Uint32()%10000)
+			}
+			remoteLocation = &remote_pb.RemoteStorageLocation{
+				Name:   *option.createBucketAt,
+				Bucket: bucketName,
+				Path:   "/",
+			}
+			// need to add new mapping here before getting updates from metadata tailing
+			option.mappings.Mappings[string(bucketPath)] = remoteLocation
+		} else {
+			bucketName = remoteLocation.Bucket
 		}

 		glog.V(0).Infof("create bucket %s", bucketName)
@@ -88,16 +113,6 @@ func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.
 			return fmt.Errorf("create bucket %s in %s: %v", bucketName, remoteConf.Name, err)
 		}

-		bucketPath := util.FullPath(option.bucketsDir).Child(entry.Name)
-		remoteLocation := &remote_pb.RemoteStorageLocation{
-			Name:   *option.createBucketAt,
-			Bucket: bucketName,
-			Path:   "/",
-		}
-
-		// need to add new mapping here before getting upates from metadata tailing
-		option.mappings.Mappings[string(bucketPath)] = remoteLocation
-
 		return filer.InsertMountMapping(option, string(bucketPath), remoteLocation)

 	}
@@ -159,13 +174,16 @@ func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.
 			return handleEtcRemoteChanges(resp)
 		}

-		if message.OldEntry == nil && message.NewEntry == nil {
+		if filer_pb.IsEmpty(resp) {
 			return nil
 		}
-		if message.OldEntry == nil && message.NewEntry != nil {
+		if filer_pb.IsCreate(resp) {
 			if message.NewParentPath == option.bucketsDir {
 				return handleCreateBucket(message.NewEntry)
 			}
+			if strings.HasPrefix(message.NewParentPath, option.bucketsDir) && strings.Contains(message.NewParentPath, "/.uploads/") {
+				return nil
+			}
 			if !filer.HasData(message.NewEntry) {
 				return nil
 			}
@@ -188,14 +206,13 @@ func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.
 				return client.WriteDirectory(dest, message.NewEntry)
 			}
 			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
-			reader := filer.NewFileReader(filerSource, message.NewEntry)
-			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
+			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
 			if writeErr != nil {
 				return writeErr
 			}
 			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
 		}
-		if message.OldEntry != nil && message.NewEntry == nil {
+		if filer_pb.IsDelete(resp) {
 			if resp.Directory == option.bucketsDir {
 				return handleDeleteBucket(message.OldEntry)
 			}
@@ -247,15 +264,13 @@ func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.
 						// update directory property
 						return nil
 					}
-					if filer.IsSameData(message.OldEntry, message.NewEntry) {
+					if message.OldEntry.RemoteEntry != nil && filer.IsSameData(message.OldEntry, message.NewEntry) {
 						glog.V(2).Infof("update meta: %+v", resp)
 						oldDest := toRemoteStorageLocation(oldBucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), oldRemoteStorageMountLocation)
 						return client.UpdateFileMetadata(oldDest, message.OldEntry, message.NewEntry)
 					} else {
 						newDest := toRemoteStorageLocation(newBucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), newRemoteStorageMountLocation)
-						reader := filer.NewFileReader(filerSource, message.NewEntry)
-						glog.V(0).Infof("create %s", remote_storage.FormatLocation(newDest))
-						remoteEntry, writeErr := client.WriteFile(newDest, message.NewEntry, reader)
+						remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, newDest)
 						if writeErr != nil {
 							return writeErr
 						}
@@ -292,9 +307,7 @@ func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.
 				if message.NewEntry.IsDirectory {
 					return client.WriteDirectory(newDest, message.NewEntry)
 				}
-				reader := filer.NewFileReader(filerSource, message.NewEntry)
-				glog.V(0).Infof("create %s", remote_storage.FormatLocation(newDest))
-				remoteEntry, writeErr := client.WriteFile(newDest, message.NewEntry, reader)
+				remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, newDest)
 				if writeErr != nil {
 					return writeErr
 				}
@@ -307,7 +320,7 @@ func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.
 	return eachEntryFunc, nil
 }

-func (option *RemoteSyncOptions) findRemoteStorageClient(bucketName string) (client remote_storage.RemoteStorageClient, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, err error) {
+func (option *RemoteGatewayOptions) findRemoteStorageClient(bucketName string) (client remote_storage.RemoteStorageClient, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, err error) {
 	bucket := util.FullPath(option.bucketsDir).Child(bucketName)

 	var isMounted bool
@@ -327,7 +340,7 @@ func (option *RemoteSyncOptions) findRemoteStorageClient(bucketName string) (cli
 	return client, remoteStorageMountLocation, nil
 }

-func (option *RemoteSyncOptions) detectBucketInfo(actualDir string) (bucket util.FullPath, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, remoteConf *remote_pb.RemoteConf, ok bool) {
+func (option *RemoteGatewayOptions) detectBucketInfo(actualDir string) (bucket util.FullPath, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, remoteConf *remote_pb.RemoteConf, ok bool) {
 	bucket, ok = extractBucketPath(option.bucketsDir, actualDir)
 	if !ok {
 		return "", nil, nil, false
@@ -355,9 +368,9 @@ func extractBucketPath(bucketsDir, dir string) (util.FullPath, bool) {
 	return util.FullPath(bucketsDir).Child(parts[0]), true
 }

-func (option *RemoteSyncOptions) collectRemoteStorageConf() (err error) {
+func (option *RemoteGatewayOptions) collectRemoteStorageConf() (err error) {

-	if mappings, err := filer.ReadMountMappings(option.grpcDialOption, *option.filerAddress); err != nil {
+	if mappings, err := filer.ReadMountMappings(option.grpcDialOption, pb.ServerAddress(*option.filerAddress)); err != nil {
 		return err
 	} else {
 		option.mappings = mappings
--- a/Show More
+++ b/Show More