3.02

fix filer helm pvc configuration

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+      interval: "weekly"
+- package-ecosystem: gomod
+  directory: "/"
+  schedule:
+    interval: weekly

.github/workflows/binaries_dev.yml

@@ -4,15 +4,20 @@ on:
   push:
     branches: [ master ]
 
+permissions:
+  contents: read
+
 jobs:
 
   cleanup:
+    permissions:
+      contents: write  # for mknejp/delete-release-assets to delete release assets
     runs-on: ubuntu-latest
 
     steps:
 
       - name: Delete old release assets
-        uses: mknejp/delete-release-assets@v1
+        uses: mknejp/delete-release-assets@a8aaab13272b1eaac16cc46dddd3f725b97ee05a # v1
         with:
           token: ${{ github.token }}
           tag: dev
@@ -21,6 +26,8 @@ jobs:
             weed-*
 
   build_dev_linux_windows:
+    permissions:
+      contents: write  # for wangyoucao577/go-release-action to upload release assets
     needs: cleanup
     runs-on: ubuntu-latest
     strategy:
@@ -31,13 +38,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -53,7 +60,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -68,6 +75,8 @@ jobs:
           asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
   build_dev_darwin:
+    permissions:
+      contents: write  # for wangyoucao577/go-release-action to upload release assets
     needs: build_dev_linux_windows
     runs-on: ubuntu-latest
     strategy:
@@ -78,13 +87,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -100,7 +109,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release0.yml

@@ -11,9 +11,14 @@ on:
   workflow_dispatch:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
+permissions:
+  contents: read
+
 jobs:
 
   build-release-binaries_windows:
+    permissions:
+      contents: write  # for wangyoucao577/go-release-action to upload release assets
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -23,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -39,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release1.yml

@@ -11,9 +11,14 @@ on:
   workflow_dispatch:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
+permissions:
+  contents: read
+
 jobs:
 
   build-release-binaries_linux:
+    permissions:
+      contents: write  # for wangyoucao577/go-release-action to upload release assets
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -23,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -39,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release2.yml

@@ -11,9 +11,14 @@ on:
   workflow_dispatch:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
+permissions:
+  contents: read
+
 jobs:
 
   build-release-binaries_darwin:
+    permissions:
+      contents: write  # for wangyoucao577/go-release-action to upload release assets
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -23,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -39,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release3.yml

@@ -11,9 +11,14 @@ on:
   workflow_dispatch:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
+permissions:
+  contents: read
+
 jobs:
 
   build-release-binaries_freebsd:
+    permissions:
+      contents: write  # for wangyoucao577/go-release-action to upload release assets
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -23,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -39,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@v1.22
+        uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/container_dev.yml

@@ -5,6 +5,9 @@ on:
     branches: [ master ]
   workflow_dispatch: []
 
+permissions:
+  contents: read
+
 jobs:
 
   build-dev-containers:
@@ -13,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@b2391d37b4157fa4aa2e118d643f417910ff3242 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -30,30 +33,30 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@f211e3e9ded2d9377c8cadc4489a4e38014bc4c9 # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_latest.yml

@@ -6,6 +6,9 @@ on:
       - '*'
   workflow_dispatch: []
 
+permissions:
+  contents: read
+
 jobs:
 
   build-latest-container:
@@ -14,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@b2391d37b4157fa4aa2e118d643f417910ff3242 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -31,30 +34,30 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@f211e3e9ded2d9377c8cadc4489a4e38014bc4c9 # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release1.yml

@@ -6,6 +6,9 @@ on:
       - '*'
   workflow_dispatch: []
 
+permissions:
+  contents: read
+
 jobs:
   build-default-release-container:
     runs-on: [ubuntu-latest]
@@ -13,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@b2391d37b4157fa4aa2e118d643f417910ff3242 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -31,20 +34,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@f211e3e9ded2d9377c8cadc4489a4e38014bc4c9 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release2.yml

@@ -6,6 +6,9 @@ on:
       - '*'
   workflow_dispatch: []
 
+permissions:
+  contents: read
+
 jobs:
 
   build-large-release-container:
@@ -14,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@b2391d37b4157fa4aa2e118d643f417910ff3242 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -32,20 +35,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@f211e3e9ded2d9377c8cadc4489a4e38014bc4c9 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release3.yml

@@ -6,6 +6,9 @@ on:
       - '*'
   workflow_dispatch: []
 
+permissions:
+  contents: read
+
 jobs:
 
   build-large-release-container_rocksdb:
@@ -14,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@b2391d37b4157fa4aa2e118d643f417910ff3242 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -32,20 +35,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@f211e3e9ded2d9377c8cadc4489a4e38014bc4c9 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/depsreview.yml

@@ -0,0 +1,14 @@
+name: 'Dependency Review'
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@3f943b86c9a289f4e632c632695e2e0898d9d67d

.github/workflows/go.yml

@@ -10,6 +10,9 @@ concurrency:
   group: ${{ github.head_ref }}/go
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
 
   build:
@@ -18,24 +21,20 @@ jobs:
     steps:
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@f6164bd8c8acb4a71fb2791a8b6c4024ff038dab # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
 
     - name: Get dependencies
       run: |
         cd weed; go get -v -t -d ./...
-        if [ -f Gopkg.toml ]; then
-            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-            dep ensure
-        fi
 
     - name: Build
-      run: cd weed; go build -v .
+      run: cd weed; go build -tags "elastic gocdk sqlite hdfs ydb" -v .
 
     - name: Test
-      run: cd weed; go test -v ./...
+      run: cd weed; go test -tags "elastic gocdk sqlite hdfs ydb" -v ./...

.gitignore

@@ -55,6 +55,8 @@ Temporary Items
 # Mongo Explorer plugin:
 # .idea/mongoSettings.xml
 
+## vscode
+.vscode
 ## File-based project format:
 *.ipr
 *.iws
@@ -75,6 +77,8 @@ com_crashlytics_export_strings.xml
 crashlytics.properties
 crashlytics-build.properties
 
+workspace/
+
 test_data
 build
 target

Makefile

@@ -0,0 +1,11 @@
+BINARY = weed
+
+SOURCE_DIR = .
+
+all: install
+
+install:
+	cd weed; go install
+
+full_install:
+	cd weed; go install -tags "elastic gocdk sqlite hdfs ydb"

README.md

@@ -33,7 +33,7 @@ Your support will be really appreciated by me and other supporters!
 
 
 ### Gold Sponsors
-![shuguang](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/shuguang.png)
+- [![nodion](https://www.nodion.com/img/logo.svg)](https://www.nodion.com)
 
 ---
 
@@ -42,6 +42,7 @@ Your support will be really appreciated by me and other supporters!
 - [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
 - [SeaweedFS on Twitter](https://twitter.com/SeaweedFS)
 - [SeaweedFS on Telegram](https://t.me/Seaweedfs) 
+- [SeaweedFS on Reddit](https://www.reddit.com/r/SeaweedFS/)
 - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
 - [SeaweedFS White Paper](https://github.com/chrislusf/seaweedfs/wiki/SeaweedFS_Architecture.pdf)
@@ -54,6 +55,7 @@ Table of Contents
 * [Quick Start](#quick-start)
     * [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
     * [Quick Start with Single Binary](#quick-start-with-single-binary)
+    * [Quick Start SeaweedFS S3 on AWS](#quick-start-seaweedfs-s3-on-aws)
 * [Introduction](#introduction)
 * [Features](#features)
     * [Additional Features](#additional-features)
@@ -82,6 +84,9 @@ Table of Contents
 
 Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -mserver="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
 
+## Quick Start SeaweedFS S3 on AWS ##
+* Setup fast production-ready [SeaweedFS S3 on AWS with cloudformation](https://aws.amazon.com/marketplace/pp/prodview-nzelz5gprlrjc)
+
 ## Introduction ##
 
 SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
@@ -105,7 +110,7 @@ Also, SeaweedFS implements erasure coding with ideas from
 
 On top of the object store, optional [Filer] can support directories and POSIX attributes. 
 Filer is a separate linearly-scalable stateless server with customizable metadata stores, 
-e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, Sqlite, MemSql, TiDB, Etcd, CockroachDB, etc.
+e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, Sqlite, MemSql, TiDB, Etcd, CockroachDB, YDB, etc.
 
 For any distributed key value stores, the large values can be offloaded to SeaweedFS. 
 With the fast access speed and linearly scalable capacity, 
@@ -407,7 +412,7 @@ The architectures are mostly the same. SeaweedFS aims to store and read files fa
 
 * SeaweedFS optimizes for small files, ensuring O(1) disk seek operation, and can also handle large files.
 * SeaweedFS statically assigns a volume id for a file. Locating file content becomes just a lookup of the volume id, which can be easily cached.
-* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Redis, Cassandra, HBase, Mongodb, Elastic Search, MySql, Postgres, Sqlite, MemSql, TiDB, CockroachDB, Etcd etc, and is easy to customized.
+* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Redis, Cassandra, HBase, Mongodb, Elastic Search, MySql, Postgres, Sqlite, MemSql, TiDB, CockroachDB, Etcd, YDB etc, and is easy to customized.
 * SeaweedFS Volume server also communicates directly with clients via HTTP, supporting range queries, direct uploads, etc.
 
 | System         | File Metadata                   | File Content Read| POSIX  | REST API | Optimized for large number of small files |
@@ -449,7 +454,7 @@ Ceph uses CRUSH hashing to automatically manage the data placement, which is eff
 
 SeaweedFS is optimized for small files. Small files are stored as one continuous block of content, with at most 8 unused bytes between files. Small file access is O(1) disk read.
 
-SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mongodb, Redis, Elastic Search, Cassandra, HBase, MemSql, TiDB, CockroachCB, Etcd, to manage file directories. These stores are proven, scalable, and easier to manage.
+SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mongodb, Redis, Elastic Search, Cassandra, HBase, MemSql, TiDB, CockroachCB, Etcd, YDB, to manage file directories. These stores are proven, scalable, and easier to manage.
 
 | SeaweedFS         | comparable to Ceph | advantage |
 | -------------  | ------------- | ---------------- |

docker/Dockerfile.go_build

@@ -1,4 +1,4 @@
-FROM golang:1.17-alpine as builder
+FROM golang:1.18-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs

docker/Dockerfile.go_build_large

@@ -1,4 +1,4 @@
-FROM golang:1.17-alpine as builder
+FROM golang:1.18-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/chrislusf/
 RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs

docker/Dockerfile.rocksdb_large

@@ -1,9 +1,9 @@
-FROM golang:1.17-buster as builder
+FROM golang:1.18-buster as builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v6.22.1
+ENV ROCKSDB_VERSION v7.0.4
 
 # build RocksDB
 RUN cd /tmp && \

docker/Makefile

@@ -49,18 +49,30 @@ dev_replicate: build
 dev_auditlog: build
 	docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up
 
+dev_ydb: build
+	docker-compose -f compose/local-ydb-compose.yml -p seaweedfs up
+
 cluster: build
 	docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up
 
 2clusters: build
 	docker-compose -f compose/local-clusters-compose.yml -p seaweedfs up
 
+2mount: build
+	docker-compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
+
+hashicorp_raft: build
+	docker-compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
+
 s3tests: build s3tests_build
 	docker-compose -f compose/local-s3tests-compose.yml -p seaweedfs up
 
 filer_etcd: build
 	docker stack deploy -c compose/swarm-etcd.yml fs
 
+test_etcd: build
+	docker-compose -f compose/test-etcd-filer.yml -p seaweedfs up
+
 clean:
 	rm ./weed
 
@@ -74,4 +86,4 @@ certstrap:
 	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" volume01.dev || true
 	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" master01.dev || true
 	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" filer01.dev || true
-	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" client01.dev || true
+	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" client01.dev || true

docker/compose/local-cluster-compose.yml

@@ -36,7 +36,7 @@ services:
     ports:
       - 8080:8080
       - 18080:18080
-    command: 'volume -dataCenter dc1 -rack v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
+    command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
     depends_on:
       - master0
       - master1
@@ -46,7 +46,7 @@ services:
     ports:
       - 8082:8082
       - 18082:18082
-    command: 'volume -dataCenter dc2 -rack v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
+    command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
     depends_on:
       - master0
       - master1
@@ -56,7 +56,7 @@ services:
     ports:
       - 8083:8083
       - 18083:18083
-    command: 'volume -dataCenter dc3 -rack v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
+    command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
     depends_on:
       - master0
       - master1
@@ -67,7 +67,7 @@ services:
       - 8888:8888
       - 18888:18888
       - 8111:8111
-    command: 'filer -defaultReplicaPlacement 100 -iam -master="master0:9333,master1:9334,master2:9335"'
+    command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335"'
     depends_on:
       - master0
       - master1
@@ -85,4 +85,4 @@ services:
       - master2
       - volume1
       - volume2
-      - filer
+      - filer

docker/compose/local-hashicorp-raft-compose.yml

@@ -0,0 +1,89 @@
+version: '2'
+
+services:
+  master0:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
+    volumes:
+      - ./master/0:/data
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+  master1:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9334:9334
+      - 19334:19334
+    command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
+    volumes:
+      - ./master/1:/data
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+  master2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9335:9335
+      - 19335:19335
+    command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
+    volumes:
+      - ./master/2:/data
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+  volume1:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
+    depends_on:
+      - master0
+      - master1
+  volume2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8082:8082
+      - 18082:18082
+    command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
+    depends_on:
+      - master0
+      - master1
+  volume3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8083:8083
+      - 18083:18083
+    command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
+    depends_on:
+      - master0
+      - master1
+  filer:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+      - 8111:8111
+    command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335"'
+    depends_on:
+      - master0
+      - master1
+      - volume1
+      - volume2
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8333:8333
+    command: '-v=9 s3 -ip.bind="s3" -filer="filer:8888"'
+    depends_on:
+      - master0
+      - master1
+      - volume1
+      - volume2
+      - filer

docker/compose/local-s3tests-compose.yml

@@ -24,7 +24,7 @@ services:
       - 8888:8888
       - 18888:18888
       - 8000:8000
-    command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000'
+    command: 'filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false'
     volumes:
       - ./s3.json:/etc/seaweedfs/s3.json
     depends_on:
@@ -38,7 +38,7 @@ services:
       S3TEST_CONF: "s3tests.conf"
       NOSETESTS_OPTIONS: "--verbose --logging-level=ERROR --with-xunit --failure-detail s3tests_boto3.functional.test_s3"
       NOSETESTS_ATTR: "!tagging,!fails_on_aws,!encryption,!bucket-policy,!versioning,!fails_on_rgw,!bucket-policy,!fails_with_subdomain,!policy_status,!object-lock,!lifecycle,!cors,!user-policy"
-      NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_notexist|bucketv2_notexist|bucket_delete_nonempty|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|multipart_upload_incorrect_etag|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket)"
+      NOSETESTS_EXCLUDE: "(get_bucket_encryption|put_bucket_encryption|bucket_list_delimiter_basic|bucket_listv2_delimiter_basic|bucket_listv2_encoding_basic|bucket_list_encoding_basic|bucket_list_delimiter_prefix|bucket_listv2_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_prefix_ends_with_delimiter|bucket_list_delimiter_alt|bucket_listv2_delimiter_alt|bucket_list_delimiter_prefix_underscore|bucket_list_delimiter_percentage|bucket_listv2_delimiter_percentage|bucket_list_delimiter_whitespace|bucket_listv2_delimiter_whitespace|bucket_list_delimiter_dot|bucket_listv2_delimiter_dot|bucket_list_delimiter_unreadable|bucket_listv2_delimiter_unreadable|bucket_listv2_fetchowner_defaultempty|bucket_listv2_fetchowner_empty|bucket_list_prefix_delimiter_alt|bucket_listv2_prefix_delimiter_alt|bucket_list_prefix_delimiter_prefix_not_exist|bucket_listv2_prefix_delimiter_prefix_not_exist|bucket_list_prefix_delimiter_delimiter_not_exist|bucket_listv2_prefix_delimiter_delimiter_not_exist|bucket_list_prefix_delimiter_prefix_delimiter_not_exist|bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist|bucket_list_maxkeys_none|bucket_listv2_maxkeys_none|bucket_list_maxkeys_invalid|bucket_listv2_continuationtoken_empty|bucket_list_return_data|bucket_list_objects_anonymous|bucket_listv2_objects_anonymous|bucket_concurrent_set_canned_acl|object_write_to_nonexist_bucket|object_requestid_matches_header_on_error|object_set_get_metadata_none_to_good|object_set_get_metadata_none_to_empty|object_set_get_metadata_overwrite_to_empty|post_object_anonymous_request|post_object_authenticated_request|post_object_authenticated_no_content_type|post_object_authenticated_request_bad_access_key|post_object_set_success_code|post_object_set_invalid_success_code|post_object_upload_larger_than_chunk|post_object_set_key_from_filename|post_object_ignored_header|post_object_case_insensitive_condition_fields|post_object_escaped_field_values|post_object_success_redirect_action|post_object_invalid_signature|post_object_invalid_access_key|post_object_missing_policy_condition|post_object_user_specified_header|post_object_request_missing_policy_specified_field|post_object_expired_policy|post_object_invalid_request_field_value|get_object_ifunmodifiedsince_good|put_object_ifmatch_failed|object_raw_get_bucket_gone|object_delete_key_bucket_gone|object_raw_get_bucket_acl|object_raw_get_object_acl|object_raw_response_headers|object_raw_authenticated_bucket_gone|object_raw_get_x_amz_expires_out_max_range|object_raw_get_x_amz_expires_out_positive_range|object_anon_put_write_access|object_raw_put_authenticated_expired|bucket_create_exists|bucket_create_naming_bad_short_one|bucket_create_naming_bad_short_two|bucket_get_location|bucket_acl_default|bucket_acl_canned|bucket_acl_canned_publicreadwrite|bucket_acl_canned_authenticatedread|object_acl_default|object_acl_canned_during_create|object_acl_canned|object_acl_canned_publicreadwrite|object_acl_canned_authenticatedread|object_acl_canned_bucketownerread|object_acl_canned_bucketownerfullcontrol|object_acl_full_control_verify_attributes|bucket_acl_canned_private_to_private|bucket_acl_grant_nonexist_user|bucket_acl_no_grants|bucket_acl_grant_email_not_exist|bucket_acl_revoke_all|bucket_recreate_not_overriding|object_copy_verify_contenttype|object_copy_to_itself_with_metadata|object_copy_not_owned_bucket|object_copy_not_owned_object_bucket|object_copy_retaining_metadata|object_copy_replacing_metadata|multipart_upload_empty|multipart_copy_invalid_range|multipart_copy_special_names|multipart_upload_resend_part|multipart_upload_size_too_small|abort_multipart_upload_not_found|multipart_upload_missing_part|100_continue|ranged_request_invalid_range|ranged_request_empty_object|access_bucket|list_multipart_upload_owner|multipart_upload_small)"
     depends_on:
       - master
       - volume

docker/compose/local-sync-mount-compose.yml

@@ -0,0 +1,21 @@
+version: '3.9'
+services:
+  node1:
+    image: chrislusf/seaweedfs:local
+    command: "server -master -volume -filer"
+  mount1:
+    image: chrislusf/seaweedfs:local
+    privileged: true
+    command: "mount -filer=node1:8888 -dir=/mnt -dirAutoCreate"
+  node2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 7888:8888
+    command: "server -master -volume -filer"
+  mount2:
+    image: chrislusf/seaweedfs:local
+    privileged: true
+    command: "mount -filer=node2:8888 -dir=/mnt -dirAutoCreate"
+  sync:
+    image: chrislusf/seaweedfs:local
+    command: "-v=4 filer.sync -a=node1:8888 -b=node2:8888 -a.debug -b.debug"

docker/compose/local-ydb-compose.yml

@@ -0,0 +1,35 @@
+version: '2'
+
+services:
+  ydb:
+    image: cr.yandex/yc/yandex-docker-local-ydb
+    ports:
+      - 2135:2135
+      - 8765:8765
+      - 2136:2136
+    environment:
+      - YDB_DEFAULT_LOG_LEVEL=DEBUG
+      - GRPC_TLS_PORT=2135
+      - GRPC_PORT=2136
+      - MON_PORT=8765
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+      - 8084:8080
+      - 18084:18080
+      - 8888:8888
+      - 8000:8000
+      - 18888:18888
+    command: "server -ip=s3 -filer -master.volumeSizeLimitMB=16 -volume.max=0 -volume -volume.preStopSeconds=1 -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false"
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
+    environment:
+      WEED_LEVELDB2_ENABLED: "false"
+      WEED_YDB_ENABLED: "true"
+      WEED_YDB_DSN: "grpc://ydb:2136/?database=local"
+      WEED_YDB_PREFIX: "seaweedfs"
+      YDB_ANONYMOUS_CREDENTIALS: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1

docker/compose/master-cloud.toml

@@ -28,3 +28,4 @@ sleep_minutes = 17          # sleep minutes between each script execution
 	region = "us-east-2"
 	bucket = "volume_bucket"    # an existing bucket
 	endpoint = "http://server2:8333"
+	storage_class = "STANDARD_IA"

docker/compose/test-etcd-filer.yml

@@ -0,0 +1,61 @@
+version: '2'
+
+services:
+  etcd:
+    image: quay.io/coreos/etcd:v3.5.4
+    command: "etcd --advertise-client-urls http://etcd:2379 --listen-client-urls http://0.0.0.0:2379"
+    ports:
+      - 2379:2379
+  master:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "master -ip=master -volumeSizeLimitMB=1024"
+  volume:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: "volume -mserver=master:9333 -port=8080 -ip=volume -max=0 -preStopSeconds=1"
+    depends_on:
+      - master
+  s3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+      - 8333:8333
+    command: '-v 9 filer -master="master:9333" -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333'
+    environment:
+      WEED_LEVELDB2_ENABLED: 'false'
+      WEED_ETCD_ENABLED: 'true'
+      WEED_ETCD_SERVERS: "http://etcd:2379"
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
+    depends_on:
+      - etcd
+      - master
+      - volume
+  registry:
+    image: registry:2
+    environment:
+      REGISTRY_HTTP_ADDR: "0.0.0.0:5001" # seaweedfs s3
+      REGISTRY_LOG_LEVEL: "debug"
+      REGISTRY_STORAGE: "s3"
+      REGISTRY_STORAGE_S3_REGION: "us-east-1"
+      REGISTRY_STORAGE_S3_REGIONENDPOINT: "http://s3:8333"
+      REGISTRY_STORAGE_S3_BUCKET: "registry"
+      REGISTRY_STORAGE_S3_ACCESSKEY: "some_access_key1"
+      REGISTRY_STORAGE_S3_SECRETKEY: "some_secret_key1"
+      REGISTRY_STORAGE_S3_V4AUTH: "true"
+      REGISTRY_STORAGE_S3_SECURE: "false"
+      REGISTRY_STORAGE_S3_SKIPVERIFY: "true"
+      REGISTRY_STORAGE_S3_ROOTDIRECTORY: "/"
+      REGISTRY_STORAGE_DELETE_ENABLED: "true"
+      REGISTRY_STORAGE_REDIRECT_DISABLE: "true"
+      REGISTRY_VALIDATION_DISABLED: "true"
+    ports:
+      - 5001:5001
+    depends_on:
+      - s3

docker/seaweedfs-compose.yml

@@ -6,14 +6,14 @@ services:
     ports:
       - 9333:9333
       - 19333:19333
-    command: "master -ip=master"
+    command: "master -ip=master -ip.bind=0.0.0.0"
   volume:
     image: chrislusf/seaweedfs # use a remote image
     ports:
       - 8080:8080
       - 18080:18080
       - 9325:9325
-    command: 'volume -mserver="master:9333" -port=8080  -metricsPort=9325'
+    command: 'volume -mserver="master:9333" -ip.bind=0.0.0.0 -port=8080  -metricsPort=9325'
     depends_on:
       - master
   filer:
@@ -22,7 +22,7 @@ services:
       - 8888:8888
       - 18888:18888
       - 9326:9326
-    command: 'filer -master="master:9333"  -metricsPort=9326'
+    command: 'filer -master="master:9333" -ip.bind=0.0.0.0 -metricsPort=9326'
     tty: true
     stdin_open: true
     depends_on:
@@ -33,7 +33,7 @@ services:
     ports:
       - 8333:8333
       - 9327:9327
-    command: 's3 -filer="filer:8888" -metricsPort=9327'
+    command: 's3 -filer="filer:8888" -ip.bind=0.0.0.0 -metricsPort=9327'
     depends_on:
       - master
       - volume

go.mod

@@ -1,23 +1,23 @@
 module github.com/chrislusf/seaweedfs
 
-go 1.17
+go 1.18
 
 require (
-	cloud.google.com/go v0.94.1 // indirect
-	cloud.google.com/go/pubsub v1.3.1
-	cloud.google.com/go/storage v1.16.1
+	cloud.google.com/go v0.100.2 // indirect
+	cloud.google.com/go/pubsub v1.21.0
+	cloud.google.com/go/storage v1.22.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.14.0
-	github.com/OneOfOne/xxhash v1.2.2
-	github.com/Shopify/sarama v1.23.1
-	github.com/aws/aws-sdk-go v1.35.3
+	github.com/OneOfOne/xxhash v1.2.8
+	github.com/Shopify/sarama v1.32.0
+	github.com/aws/aws-sdk-go v1.43.45
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cespare/xxhash v1.1.0
-	github.com/cespare/xxhash/v2 v2.1.1 // indirect
-	github.com/chrislusf/raft v1.0.7
-	github.com/colinmarc/hdfs/v2 v2.2.0
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/chrislusf/raft v1.0.8
+	github.com/colinmarc/hdfs/v2 v2.3.0
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -32,85 +32,77 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
-	github.com/fclairamb/ftpserverlib v0.8.0
-	github.com/frankban/quicktest v1.7.2 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/fclairamb/ftpserverlib v0.18.0
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
-	github.com/go-redis/redis/v8 v8.4.4
-	github.com/go-redsync/redsync/v4 v4.4.1
-	github.com/go-sql-driver/mysql v1.5.0
-	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/go-redis/redis/v8 v8.11.5
+	github.com/go-redsync/redsync/v4 v4.5.0
+	github.com/go-sql-driver/mysql v1.6.0
+	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/go-zookeeper/zk v1.0.2 // indirect
 	github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
-	github.com/golang-jwt/jwt v3.2.1+incompatible
+	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/btree v1.0.0
-	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/btree v1.0.1
+	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/uuid v1.3.0
-	github.com/google/wire v0.4.0 // indirect
-	github.com/googleapis/gax-go v2.0.2+incompatible // indirect
-	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
-	github.com/gorilla/mux v1.7.4
-	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
+	github.com/google/wire v0.5.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.3.0 // indirect
+	github.com/gorilla/mux v1.8.0
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.0 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jcmturner/gofork v1.0.0 // indirect
-	github.com/jcmturner/gokrb5/v8 v8.4.1
-	github.com/jinzhu/copier v0.2.8
+	github.com/jcmturner/gokrb5/v8 v8.4.2
+	github.com/jinzhu/copier v0.3.5
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/json-iterator/go v1.1.11
-	github.com/karlseguin/ccache/v2 v2.0.7
+	github.com/json-iterator/go v1.1.12
+	github.com/karlseguin/ccache/v2 v2.0.8
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.13.6 // indirect
-	github.com/klauspost/cpuid v1.2.1 // indirect
-	github.com/klauspost/reedsolomon v1.9.2
-	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
+	github.com/klauspost/compress v1.15.1 // indirect
+	github.com/klauspost/reedsolomon v1.9.16
 	github.com/kurin/blazer v0.5.3
-	github.com/lib/pq v1.10.0
-	github.com/linxGnu/grocksdb v1.6.38
-	github.com/magiconair/properties v1.8.1 // indirect
-	github.com/mailru/easyjson v0.7.1 // indirect
-	github.com/mattn/go-ieproxy v0.0.1 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/lib/pq v1.10.5
+	github.com/linxGnu/grocksdb v1.7.1
+	github.com/magiconair/properties v1.8.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-ieproxy v0.0.3 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/mitchellh/mapstructure v1.1.2 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/nats-io/jwt v1.0.1 // indirect
-	github.com/nats-io/nats.go v1.10.0 // indirect
-	github.com/nats-io/nkeys v0.2.0 // indirect
-	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/olivere/elastic/v7 v7.0.19
-	github.com/pelletier/go-toml v1.7.0 // indirect
-	github.com/peterh/liner v1.1.0
-	github.com/pierrec/lz4 v2.2.7+incompatible // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/olivere/elastic/v7 v7.0.32
+	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/peterh/liner v1.2.2
+	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/posener/complete v1.2.3
 	github.com/pquerna/cachecontrol v0.1.0
-	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.26.0 // indirect
-	github.com/prometheus/procfs v0.6.0 // indirect
-	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
-	github.com/seaweedfs/goexif v1.0.2
-	github.com/sirupsen/logrus v1.6.0 // indirect
+	github.com/seaweedfs/goexif v2.0.0+incompatible
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/afero v1.6.0 // indirect
-	github.com/spf13/cast v1.3.0 // indirect
+	github.com/spf13/afero v1.8.2 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.4.0
-	github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
-	github.com/stretchr/testify v1.7.0
+	github.com/spf13/viper v1.11.0
+	github.com/streadway/amqp v1.0.0
+	github.com/stretchr/testify v1.7.1
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
-	github.com/tidwall/gjson v1.10.2
+	github.com/tidwall/gjson v1.14.1
 	github.com/tidwall/match v1.1.1
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
@@ -120,75 +112,114 @@ require (
 	github.com/viant/ptrie v0.3.0
 	github.com/viant/toolbox v0.33.2 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.0.2 // indirect
+	github.com/xdg-go/scram v1.1.0 // indirect
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.etcd.io/etcd/client/v3 v3.5.0
-	go.mongodb.org/mongo-driver v1.8.0
+	go.etcd.io/etcd/client/v3 v3.5.4
+	go.mongodb.org/mongo-driver v1.9.0
 	go.opencensus.io v0.23.0 // indirect
-	go.opentelemetry.io/otel v0.15.0 // indirect
-	gocloud.dev v0.20.0
-	gocloud.dev/pubsub/natspubsub v0.20.0
-	gocloud.dev/pubsub/rabbitpubsub v0.20.0
-	golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect
+	gocloud.dev v0.25.0
+	gocloud.dev/pubsub/natspubsub v0.25.0
+	gocloud.dev/pubsub/rabbitpubsub v0.25.0
+	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
+	golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
-	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/sys v0.0.0-20220111092808-5a964db01320
+	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4
+	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
+	golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/api v0.57.0
+	golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023
+	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
+	google.golang.org/api v0.77.0
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 // indirect
-	google.golang.org/grpc v1.40.0
-	google.golang.org/protobuf v1.27.1
+	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
+	google.golang.org/grpc v1.46.0
+	google.golang.org/protobuf v1.28.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
-	gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
-	gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
-	gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
-	gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
 	modernc.org/b v1.0.0 // indirect
-	modernc.org/cc/v3 v3.33.5 // indirect
-	modernc.org/ccgo/v3 v3.9.4 // indirect
-	modernc.org/libc v1.9.5 // indirect
-	modernc.org/mathutil v1.2.2 // indirect
-	modernc.org/memory v1.0.4 // indirect
+	modernc.org/cc/v3 v3.35.26 // indirect
+	modernc.org/ccgo/v3 v3.16.2 // indirect
+	modernc.org/libc v1.15.0 // indirect
+	modernc.org/mathutil v1.4.1 // indirect
+	modernc.org/memory v1.0.7 // indirect
 	modernc.org/opt v0.1.1 // indirect
-	modernc.org/sqlite v1.10.7
-	modernc.org/strutil v1.1.0 // indirect
+	modernc.org/sqlite v1.17.0
+	modernc.org/strutil v1.1.1 // indirect
 	modernc.org/token v1.0.0 // indirect
 )
 
 require (
-	github.com/fluent/fluent-logger-golang v1.8.0
+	github.com/Jille/raft-grpc-transport v1.2.0
+	github.com/arangodb/go-driver v1.3.1
+	github.com/fluent/fluent-logger-golang v1.9.0
 	github.com/hanwen/go-fuse/v2 v2.1.0
+	github.com/hashicorp/raft v1.3.9
+	github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0
 )
 
 require (
-	cloud.google.com/go/kms v1.0.0 // indirect
-	github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
+	cloud.google.com/go/compute v1.6.0 // indirect
+	cloud.google.com/go/iam v0.3.0 // indirect
+	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
+	github.com/armon/go-metrics v0.3.10 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.17.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
+	github.com/aws/smithy-go v1.11.2 // indirect
+	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
+	github.com/fatih/color v1.13.0 // indirect
+	github.com/fclairamb/go-log v0.3.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/googleapis/go-type-adapters v1.0.0 // indirect
+	github.com/hashicorp/go-hclog v1.2.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-msgpack v1.1.5 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
-	github.com/jcmturner/rpc/v2 v2.0.2 // indirect
+	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.6 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-runewidth v0.0.7 // indirect
 	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
+	github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect
+	github.com/nats-io/nkeys v0.3.0 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
 	github.com/philhofer/fwd v1.1.1 // indirect
+	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/tinylib/msgp v1.1.6 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.0 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.7.0 // indirect
-	go.uber.org/zap v1.17.0 // indirect
-	golang.org/x/mod v0.4.2 // indirect
+	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc // indirect
+	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.0 // indirect
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.24.2 // indirect
+	github.com/ydb-platform/ydb-go-yc v0.6.1 // indirect
+	github.com/ydb-platform/ydb-go-yc-metadata v0.0.9 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.4 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	go.uber.org/zap v1.21.0 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	lukechampine.com/uint128 v1.1.1 // indirect
 )
 
 // replace github.com/chrislusf/raft => /Users/chrislu/go/src/github.com/chrislusf/raft

k8s/helm_charts2/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.94"
-version: "2.94"
+appVersion: "3.02"
+version: "3.02"

k8s/helm_charts2/templates/filer-statefulset.yaml

@@ -154,13 +154,16 @@ spec:
               {{- end }}
               {{- end }}
               -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
-          {{- if or (.Values.global.enableSecurity) (.Values.filer.extraVolumeMounts) }}    
           volumeMounts:
             - name: seaweedfs-filer-log-volume
               mountPath: "/logs/"
             - mountPath: /etc/sw
               name: config-users
               readOnly: true
+            {{- if .Values.filer.enablePVC }}
+            - name: data-filer
+              mountPath: /data
+            {{- end }}
             {{- if .Values.global.enableSecurity }}
             - name: security-config
               readOnly: true
@@ -183,7 +186,6 @@ spec:
               mountPath: /usr/local/share/ca-certificates/client/
             {{- end }}
             {{ tpl .Values.filer.extraVolumeMounts . | nindent 12 | trim }}
-          {{- end }}
           ports:
             - containerPort: {{ .Values.filer.port }}
               name: swfs-filer
@@ -250,16 +252,18 @@ spec:
       nodeSelector:
         {{ tpl .Values.filer.nodeSelector . | indent 8 | trim }}
       {{- end }}
-{{/*  volumeClaimTemplates:*/}}
-{{/*    - metadata:*/}}
-{{/*        name: data-{{ .Release.Namespace }}*/}}
-{{/*      spec:*/}}
-{{/*        accessModes:*/}}
-{{/*          - ReadWriteOnce*/}}
-{{/*        resources:*/}}
-{{/*          requests:*/}}
-{{/*            storage: {{ .Values.filer.storage }}*/}}
-{{/*        {{- if .Values.filer.storageClass }}*/}}
-{{/*        storageClassName: {{ .Values.filer.storageClass }}*/}}
-{{/*      {{- end }}*/}}
+  {{- if .Values.filer.enablePVC }}
+  volumeClaimTemplates:
+  - metadata:
+      name: data-filer
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: {{ .Values.filer.storage }}
+      {{- if .Values.filer.storageClass }}
+      storageClassName: {{ .Values.filer.storageClass }}
+      {{- end }}
+  {{- end }}
 {{- end }}

k8s/helm_charts2/values.yaml

@@ -41,8 +41,7 @@ master:
   grpcPort: 19333
   ipBind: "0.0.0.0"
   volumePreallocate: false
-  #Master stops directing writes to oversized volumes
-  volumeSizeLimitMB: 30000
+  volumeSizeLimitMB: 1000
   loggingOverrideLevel: null
   #number of seconds between heartbeats, default 5
   pulseSeconds: null
@@ -276,6 +275,9 @@ filer:
   # Disable http request, only gRpc operations are allowed
   disableHttp: false
 
+  # enablePVC will create a pvc for filer for data persistence.
+  enablePVC: false
+
   # storage and storageClass are the settings for configuring stateful
   # storage for the master pods. storage should be set to the disk size of
   # the attached volume. storageClass is the class of storage which defaults
@@ -358,6 +360,7 @@ filer:
     WEED_MYSQL_CONNECTION_MAX_LIFETIME_SECONDS: "600"
     # enable usage of memsql as filer backend
     WEED_MYSQL_INTERPOLATEPARAMS: "true"
+    # if you want to use leveldb2, then should enable "enablePVC". or you may lose your data.
     WEED_LEVELDB2_ENABLED: "false"
     # with http DELETE, by default the filer would check whether a folder is empty.
     # recursive_delete will delete all sub folders and files, similar to "rm -Rf"

other/java/client/src/main/proto/filer.proto

@@ -48,6 +48,9 @@ service SeaweedFiler {
     rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
     }
 
+    rpc Ping (PingRequest) returns (PingResponse) {
+    }
+
     rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
     }
 
@@ -181,6 +184,7 @@ message CreateEntryRequest {
     bool o_excl = 3;
     bool is_from_other_cluster = 4;
     repeated int32 signatures = 5;
+    bool skip_check_parent_directory = 6;
 }
 
 message CreateEntryResponse {
@@ -310,6 +314,16 @@ message StatisticsResponse {
     uint64 file_count = 6;
 }
 
+message PingRequest {
+    string target = 1; // default to ping itself
+    string target_type = 2;
+}
+message PingResponse {
+    int64 start_time_ns = 1;
+    int64 remote_time_ns = 2;
+    int64 stop_time_ns = 3;
+}
+
 message GetFilerConfigurationRequest {
 }
 message GetFilerConfigurationResponse {

other/java/hdfs-over-ftp/pom.xml

@@ -36,7 +36,7 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
-            <version>3.2.2</version>
+            <version>3.2.3</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>

other/java/hdfs3/pom.xml

@@ -6,7 +6,7 @@
 
     <properties>
         <seaweedfs.client.version>2.85</seaweedfs.client.version>
-        <hadoop.version>3.1.4</hadoop.version>
+        <hadoop.version>3.2.3</hadoop.version>
     </properties>
 
     <groupId>com.github.chrislusf</groupId>

weed/cluster/cluster.go

@@ -9,11 +9,21 @@ import (
 )
 
 const (
-	MasterType = "master"
-	FilerType  = "filer"
-	BrokerType = "broker"
+	MasterType       = "master"
+	VolumeServerType = "volumeServer"
+	FilerType        = "filer"
+	BrokerType       = "broker"
 )
 
+type FilerGroup string
+type Filers struct {
+	filers  map[pb.ServerAddress]*ClusterNode
+	leaders *Leaders
+}
+type Leaders struct {
+	leaders [3]pb.ServerAddress
+}
+
 type ClusterNode struct {
 	Address   pb.ServerAddress
 	Version   string
@@ -21,42 +31,50 @@ type ClusterNode struct {
 	createdTs time.Time
 }
 
-type Leaders struct {
-	leaders [3]pb.ServerAddress
-}
-
 type Cluster struct {
-	filers       map[pb.ServerAddress]*ClusterNode
-	filersLock   sync.RWMutex
-	filerLeaders *Leaders
-	brokers      map[pb.ServerAddress]*ClusterNode
-	brokersLock  sync.RWMutex
+	filerGroup2filers map[FilerGroup]*Filers
+	filersLock        sync.RWMutex
+	brokers           map[pb.ServerAddress]*ClusterNode
+	brokersLock       sync.RWMutex
 }
 
 func NewCluster() *Cluster {
 	return &Cluster{
-		filers:       make(map[pb.ServerAddress]*ClusterNode),
-		filerLeaders: &Leaders{},
-		brokers:      make(map[pb.ServerAddress]*ClusterNode),
+		filerGroup2filers: make(map[FilerGroup]*Filers),
+		brokers:           make(map[pb.ServerAddress]*ClusterNode),
 	}
 }
 
-func (cluster *Cluster) AddClusterNode(nodeType string, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
+func (cluster *Cluster) getFilers(filerGroup FilerGroup, createIfNotFound bool) *Filers {
+	cluster.filersLock.Lock()
+	defer cluster.filersLock.Unlock()
+	filers, found := cluster.filerGroup2filers[filerGroup]
+	if !found && createIfNotFound {
+		filers = &Filers{
+			filers:  make(map[pb.ServerAddress]*ClusterNode),
+			leaders: &Leaders{},
+		}
+		cluster.filerGroup2filers[filerGroup] = filers
+	}
+	return filers
+}
+
+func (cluster *Cluster) AddClusterNode(ns, nodeType string, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
+	filerGroup := FilerGroup(ns)
 	switch nodeType {
 	case FilerType:
-		cluster.filersLock.Lock()
-		defer cluster.filersLock.Unlock()
-		if existingNode, found := cluster.filers[address]; found {
+		filers := cluster.getFilers(filerGroup, true)
+		if existingNode, found := filers.filers[address]; found {
 			existingNode.counter++
 			return nil
 		}
-		cluster.filers[address] = &ClusterNode{
+		filers.filers[address] = &ClusterNode{
 			Address:   address,
 			Version:   version,
 			counter:   1,
 			createdTs: time.Now(),
 		}
-		return cluster.ensureFilerLeaders(true, nodeType, address)
+		return cluster.ensureFilerLeaders(filers, true, filerGroup, nodeType, address)
 	case BrokerType:
 		cluster.brokersLock.Lock()
 		defer cluster.brokersLock.Unlock()
@@ -80,22 +98,34 @@ func (cluster *Cluster) AddClusterNode(nodeType string, address pb.ServerAddress
 			},
 		}
 	case MasterType:
+		return []*master_pb.KeepConnectedResponse{
+			{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsAdd:    true,
+				},
+			},
+		}
 	}
 	return nil
 }
 
-func (cluster *Cluster) RemoveClusterNode(nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
+func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
+	filerGroup := FilerGroup(ns)
 	switch nodeType {
 	case FilerType:
-		cluster.filersLock.Lock()
-		defer cluster.filersLock.Unlock()
-		if existingNode, found := cluster.filers[address]; !found {
+		filers := cluster.getFilers(filerGroup, false)
+		if filers == nil {
+			return nil
+		}
+		if existingNode, found := filers.filers[address]; !found {
 			return nil
 		} else {
 			existingNode.counter--
 			if existingNode.counter <= 0 {
-				delete(cluster.filers, address)
-				return cluster.ensureFilerLeaders(false, nodeType, address)
+				delete(filers.filers, address)
+				return cluster.ensureFilerLeaders(filers, false, filerGroup, nodeType, address)
 			}
 		}
 	case BrokerType:
@@ -119,16 +149,29 @@ func (cluster *Cluster) RemoveClusterNode(nodeType string, address pb.ServerAddr
 			}
 		}
 	case MasterType:
+		return []*master_pb.KeepConnectedResponse{
+			{
+				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+					NodeType: nodeType,
+					Address:  string(address),
+					IsAdd:    false,
+				},
+			},
+		}
 	}
 	return nil
 }
 
-func (cluster *Cluster) ListClusterNode(nodeType string) (nodes []*ClusterNode) {
+func (cluster *Cluster) ListClusterNode(filerGroup FilerGroup, nodeType string) (nodes []*ClusterNode) {
 	switch nodeType {
 	case FilerType:
+		filers := cluster.getFilers(filerGroup, false)
+		if filers == nil {
+			return
+		}
 		cluster.filersLock.RLock()
 		defer cluster.filersLock.RUnlock()
-		for _, node := range cluster.filers {
+		for _, node := range filers.filers {
 			nodes = append(nodes, node)
 		}
 	case BrokerType:
@@ -142,41 +185,48 @@ func (cluster *Cluster) ListClusterNode(nodeType string) (nodes []*ClusterNode)
 	return
 }
 
-func (cluster *Cluster) IsOneLeader(address pb.ServerAddress) bool {
-	return cluster.filerLeaders.isOneLeader(address)
+func (cluster *Cluster) IsOneLeader(filerGroup FilerGroup, address pb.ServerAddress) bool {
+	filers := cluster.getFilers(filerGroup, false)
+	if filers == nil {
+		return false
+	}
+	return filers.leaders.isOneLeader(address)
 }
 
-func (cluster *Cluster) ensureFilerLeaders(isAdd bool, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
+func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGroup FilerGroup, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
 	if isAdd {
-		if cluster.filerLeaders.addLeaderIfVacant(address) {
+		if filers.leaders.addLeaderIfVacant(address) {
 			// has added the address as one leader
 			result = append(result, &master_pb.KeepConnectedResponse{
 				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
-					NodeType: nodeType,
-					Address:  string(address),
-					IsLeader: true,
-					IsAdd:    true,
+					FilerGroup: string(filerGroup),
+					NodeType:   nodeType,
+					Address:    string(address),
+					IsLeader:   true,
+					IsAdd:      true,
 				},
 			})
 		} else {
 			result = append(result, &master_pb.KeepConnectedResponse{
 				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
-					NodeType: nodeType,
-					Address:  string(address),
-					IsLeader: false,
-					IsAdd:    true,
+					FilerGroup: string(filerGroup),
+					NodeType:   nodeType,
+					Address:    string(address),
+					IsLeader:   false,
+					IsAdd:      true,
 				},
 			})
 		}
 	} else {
-		if cluster.filerLeaders.removeLeaderIfExists(address) {
+		if filers.leaders.removeLeaderIfExists(address) {
 
 			result = append(result, &master_pb.KeepConnectedResponse{
 				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
-					NodeType: nodeType,
-					Address:  string(address),
-					IsLeader: true,
-					IsAdd:    false,
+					FilerGroup: string(filerGroup),
+					NodeType:   nodeType,
+					Address:    string(address),
+					IsLeader:   true,
+					IsAdd:      false,
 				},
 			})
 
@@ -184,8 +234,8 @@ func (cluster *Cluster) ensureFilerLeaders(isAdd bool, nodeType string, address
 			var shortestDuration int64 = math.MaxInt64
 			now := time.Now()
 			var candidateAddress pb.ServerAddress
-			for _, node := range cluster.filers {
-				if cluster.filerLeaders.isOneLeader(node.Address) {
+			for _, node := range filers.filers {
+				if filers.leaders.isOneLeader(node.Address) {
 					continue
 				}
 				duration := now.Sub(node.createdTs).Nanoseconds()
@@ -195,7 +245,7 @@ func (cluster *Cluster) ensureFilerLeaders(isAdd bool, nodeType string, address
 				}
 			}
 			if candidateAddress != "" {
-				cluster.filerLeaders.addLeaderIfVacant(candidateAddress)
+				filers.leaders.addLeaderIfVacant(candidateAddress)
 				// added a new leader
 				result = append(result, &master_pb.KeepConnectedResponse{
 					ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
@@ -209,10 +259,11 @@ func (cluster *Cluster) ensureFilerLeaders(isAdd bool, nodeType string, address
 		} else {
 			result = append(result, &master_pb.KeepConnectedResponse{
 				ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
-					NodeType: nodeType,
-					Address:  string(address),
-					IsLeader: false,
-					IsAdd:    false,
+					FilerGroup: string(filerGroup),
+					NodeType:   nodeType,
+					Address:    string(address),
+					IsLeader:   false,
+					IsAdd:      false,
 				},
 			})
 		}

weed/cluster/cluster_test.go

@@ -9,39 +9,39 @@ import (
 func TestClusterAddRemoveNodes(t *testing.T) {
 	c := NewCluster()
 
-	c.AddClusterNode("filer", pb.ServerAddress("111:1"), "23.45")
-	c.AddClusterNode("filer", pb.ServerAddress("111:2"), "23.45")
+	c.AddClusterNode("", "filer", pb.ServerAddress("111:1"), "23.45")
+	c.AddClusterNode("", "filer", pb.ServerAddress("111:2"), "23.45")
 	assert.Equal(t, []pb.ServerAddress{
 		pb.ServerAddress("111:1"),
 		pb.ServerAddress("111:2"),
-	}, c.filerLeaders.GetLeaders())
+	}, c.getFilers("", false).leaders.GetLeaders())
 
-	c.AddClusterNode("filer", pb.ServerAddress("111:3"), "23.45")
-	c.AddClusterNode("filer", pb.ServerAddress("111:4"), "23.45")
+	c.AddClusterNode("", "filer", pb.ServerAddress("111:3"), "23.45")
+	c.AddClusterNode("", "filer", pb.ServerAddress("111:4"), "23.45")
 	assert.Equal(t, []pb.ServerAddress{
 		pb.ServerAddress("111:1"),
 		pb.ServerAddress("111:2"),
 		pb.ServerAddress("111:3"),
-	}, c.filerLeaders.GetLeaders())
+	}, c.getFilers("", false).leaders.GetLeaders())
 
-	c.AddClusterNode("filer", pb.ServerAddress("111:5"), "23.45")
-	c.AddClusterNode("filer", pb.ServerAddress("111:6"), "23.45")
-	c.RemoveClusterNode("filer", pb.ServerAddress("111:4"))
+	c.AddClusterNode("", "filer", pb.ServerAddress("111:5"), "23.45")
+	c.AddClusterNode("", "filer", pb.ServerAddress("111:6"), "23.45")
+	c.RemoveClusterNode("", "filer", pb.ServerAddress("111:4"))
 	assert.Equal(t, []pb.ServerAddress{
 		pb.ServerAddress("111:1"),
 		pb.ServerAddress("111:2"),
 		pb.ServerAddress("111:3"),
-	}, c.filerLeaders.GetLeaders())
+	}, c.getFilers("", false).leaders.GetLeaders())
 
 	// remove oldest
-	c.RemoveClusterNode("filer", pb.ServerAddress("111:1"))
+	c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1"))
 	assert.Equal(t, []pb.ServerAddress{
 		pb.ServerAddress("111:6"),
 		pb.ServerAddress("111:2"),
 		pb.ServerAddress("111:3"),
-	}, c.filerLeaders.GetLeaders())
+	}, c.getFilers("", false).leaders.GetLeaders())
 
 	// remove oldest
-	c.RemoveClusterNode("filer", pb.ServerAddress("111:1"))
+	c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1"))
 
 }

weed/command/benchmark.go

@@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool {
 		defer pprof.StopCPUProfile()
 	}
 
-	b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddresses())
+	b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
 	go b.masterClient.KeepConnectedToMaster()
 	b.masterClient.WaitUntilConnected()
 

weed/command/filer.go

@@ -5,6 +5,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"runtime"
 	"time"
 
 	"google.golang.org/grpc/reflection"
@@ -13,7 +14,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/server"
+	weed_server "github.com/chrislusf/seaweedfs/weed/server"
 	stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
@@ -29,13 +30,14 @@ var (
 )
 
 type FilerOptions struct {
-	masters                 []pb.ServerAddress
+	masters                 map[string]pb.ServerAddress
 	mastersString           *string
 	ip                      *string
 	bindIp                  *string
 	port                    *int
 	portGrpc                *int
 	publicPort              *int
+	filerGroup              *string
 	collection              *string
 	defaultReplicaPlacement *string
 	disableDirListing       *bool
@@ -58,6 +60,7 @@ type FilerOptions struct {
 func init() {
 	cmdFiler.Run = runFiler // break init cycle
 	f.mastersString = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
+	f.filerGroup = cmdFiler.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
 	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
 	f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
 	f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
@@ -89,6 +92,7 @@ func init() {
 	filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
 	filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
 	filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
+	filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 
 	// start webdav on filer
 	filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
@@ -103,6 +107,7 @@ func init() {
 
 	// start iam on filer
 	filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
+	filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address")
 	filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
 }
 
@@ -170,7 +175,7 @@ func runFiler(cmd *Command, args []string) bool {
 		}()
 	}
 
-	f.masters = pb.ServerAddresses(*f.mastersString).ToAddresses()
+	f.masters = pb.ServerAddresses(*f.mastersString).ToAddressMap()
 
 	f.startFiler()
 
@@ -198,6 +203,7 @@ func (fo *FilerOptions) startFiler() {
 
 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
 		Masters:               fo.masters,
+		FilerGroup:            *fo.filerGroup,
 		Collection:            *fo.collection,
 		DefaultReplication:    *fo.defaultReplicaPlacement,
 		DisableDirListing:     *fo.disableDirListing,
@@ -219,7 +225,7 @@ func (fo *FilerOptions) startFiler() {
 	if *fo.publicPort != 0 {
 		publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
 		glog.V(0).Infoln("Start Seaweed filer server", util.Version(), "public at", publicListeningAddress)
-		publicListener, e := util.NewListener(publicListeningAddress, 0)
+		publicListener, localPublicListner, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
 		if e != nil {
 			glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
 		}
@@ -228,44 +234,62 @@ func (fo *FilerOptions) startFiler() {
 				glog.Fatalf("Volume server fail to serve public: %v", e)
 			}
 		}()
+		if localPublicListner != nil {
+			go func() {
+				if e := http.Serve(localPublicListner, publicVolumeMux); e != nil {
+					glog.Errorf("Volume server fail to serve public: %v", e)
+				}
+			}()
+		}
 	}
 
 	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.Version(), *fo.ip, *fo.port)
-	filerListener, e := util.NewListener(
-		util.JoinHostPort(*fo.bindIp, *fo.port),
+	filerListener, filerLocalListener, e := util.NewIpAndLocalListeners(
+		*fo.bindIp, *fo.port,
 		time.Duration(10)*time.Second,
 	)
 	if e != nil {
 		glog.Fatalf("Filer listener error: %v", e)
 	}
 
-	// start on local unix socket
-	if *fo.localSocket == "" {
-		*fo.localSocket = fmt.Sprintf("/tmp/seaweefs-filer-%d.sock", *fo.port)
-		if err := os.Remove(*fo.localSocket); err != nil && !os.IsNotExist(err) {
-			glog.Fatalf("Failed to remove %s, error: %s", *fo.localSocket, err.Error())
-		}
-	}
-	filerSocketListener, err := net.Listen("unix", *fo.localSocket)
-	if err != nil {
-		glog.Fatalf("Failed to listen on %s: %v", *fo.localSocket, err)
-	}
-
 	// starting grpc server
 	grpcPort := *fo.portGrpc
-	grpcL, err := util.NewListener(util.JoinHostPort(*fo.bindIp, grpcPort), 0)
+	grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0)
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
 	}
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.filer"))
 	filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
 	reflection.Register(grpcS)
+	if grpcLocalL != nil {
+		go grpcS.Serve(grpcLocalL)
+	}
 	go grpcS.Serve(grpcL)
 
 	httpS := &http.Server{Handler: defaultMux}
-	go func() {
-		httpS.Serve(filerSocketListener)
-	}()
+	if runtime.GOOS != "windows" {
+		if *fo.localSocket == "" {
+			*fo.localSocket = fmt.Sprintf("/tmp/seaweefs-filer-%d.sock", *fo.port)
+			if err := os.Remove(*fo.localSocket); err != nil && !os.IsNotExist(err) {
+				glog.Fatalf("Failed to remove %s, error: %s", *fo.localSocket, err.Error())
+			}
+		}
+		go func() {
+			// start on local unix socket
+			filerSocketListener, err := net.Listen("unix", *fo.localSocket)
+			if err != nil {
+				glog.Fatalf("Failed to listen on %s: %v", *fo.localSocket, err)
+			}
+			httpS.Serve(filerSocketListener)
+		}()
+	}
+	if filerLocalListener != nil {
+		go func() {
+			if err := httpS.Serve(filerLocalListener); err != nil {
+				glog.Errorf("Filer Fail to serve: %v", e)
+			}
+		}()
+	}
 	if err := httpS.Serve(filerListener); err != nil {
 		glog.Fatalf("Filer Fail to serve: %v", e)
 	}

weed/command/filer_copy.go

@@ -71,7 +71,7 @@ var cmdFilerCopy = &Command{
   It can copy one or a list of files or folders.
 
   If copying a whole folder recursively:
-  All files under the folder and subfolders will be copyed.
+  All files under the folder and sub folders will be copied.
   Optional parameter "-include" allows you to specify the file name patterns.
 
   If "maxMB" is set to a positive number, files larger than it would be split into chunks.

weed/command/filer_meta_tail.go

@@ -1,12 +1,9 @@
 package command
 
 import (
-	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/golang/protobuf/jsonpb"
-	jsoniter "github.com/json-iterator/go"
-	elastic "github.com/olivere/elastic/v7"
 	"os"
 	"path/filepath"
 	"strings"
@@ -124,72 +121,3 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
 
 	return true
 }
-
-type EsDocument struct {
-	Dir         string `json:"dir,omitempty"`
-	Name        string `json:"name,omitempty"`
-	IsDirectory bool   `json:"isDir,omitempty"`
-	Size        uint64 `json:"size,omitempty"`
-	Uid         uint32 `json:"uid,omitempty"`
-	Gid         uint32 `json:"gid,omitempty"`
-	UserName    string `json:"userName,omitempty"`
-	Collection  string `json:"collection,omitempty"`
-	Crtime      int64  `json:"crtime,omitempty"`
-	Mtime       int64  `json:"mtime,omitempty"`
-	Mime        string `json:"mime,omitempty"`
-}
-
-func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) {
-	entry := event.NewEntry
-	dir, name := event.NewParentPath, entry.Name
-	id := util.Md5String([]byte(util.NewFullPath(dir, name)))
-	esEntry := &EsDocument{
-		Dir:         dir,
-		Name:        name,
-		IsDirectory: entry.IsDirectory,
-		Size:        entry.Attributes.FileSize,
-		Uid:         entry.Attributes.Uid,
-		Gid:         entry.Attributes.Gid,
-		UserName:    entry.Attributes.UserName,
-		Collection:  entry.Attributes.Collection,
-		Crtime:      entry.Attributes.Crtime,
-		Mtime:       entry.Attributes.Mtime,
-		Mime:        entry.Attributes.Mime,
-	}
-	return esEntry, id
-}
-
-func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) {
-	options := []elastic.ClientOptionFunc{}
-	options = append(options, elastic.SetURL(strings.Split(servers, ",")...))
-	options = append(options, elastic.SetSniff(false))
-	options = append(options, elastic.SetHealthcheck(false))
-	client, err := elastic.NewClient(options...)
-	if err != nil {
-		return nil, err
-	}
-	return func(resp *filer_pb.SubscribeMetadataResponse) error {
-		event := resp.EventNotification
-		if event.OldEntry != nil &&
-			(event.NewEntry == nil || resp.Directory != event.NewParentPath || event.OldEntry.Name != event.NewEntry.Name) {
-			// delete or not update the same file
-			dir, name := resp.Directory, event.OldEntry.Name
-			id := util.Md5String([]byte(util.NewFullPath(dir, name)))
-			println("delete", id)
-			_, err := client.Delete().Index(esIndex).Id(id).Do(context.Background())
-			return err
-		}
-		if event.NewEntry != nil {
-			// add a new file or update the same file
-			esEntry, id := toEsEntry(event)
-			value, err := jsoniter.Marshal(esEntry)
-			if err != nil {
-				return err
-			}
-			println(string(value))
-			_, err = client.Index().Index(esIndex).Id(id).BodyJson(string(value)).Do(context.Background())
-			return err
-		}
-		return nil
-	}, nil
-}

weed/command/filer_meta_tail_elastic.go

@@ -0,0 +1,82 @@
+//go:build elastic
+// +build elastic
+
+package command
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	jsoniter "github.com/json-iterator/go"
+	elastic "github.com/olivere/elastic/v7"
+	"strings"
+)
+
+type EsDocument struct {
+	Dir         string `json:"dir,omitempty"`
+	Name        string `json:"name,omitempty"`
+	IsDirectory bool   `json:"isDir,omitempty"`
+	Size        uint64 `json:"size,omitempty"`
+	Uid         uint32 `json:"uid,omitempty"`
+	Gid         uint32 `json:"gid,omitempty"`
+	UserName    string `json:"userName,omitempty"`
+	Collection  string `json:"collection,omitempty"`
+	Crtime      int64  `json:"crtime,omitempty"`
+	Mtime       int64  `json:"mtime,omitempty"`
+	Mime        string `json:"mime,omitempty"`
+}
+
+func toEsEntry(event *filer_pb.EventNotification) (*EsDocument, string) {
+	entry := event.NewEntry
+	dir, name := event.NewParentPath, entry.Name
+	id := util.Md5String([]byte(util.NewFullPath(dir, name)))
+	esEntry := &EsDocument{
+		Dir:         dir,
+		Name:        name,
+		IsDirectory: entry.IsDirectory,
+		Size:        entry.Attributes.FileSize,
+		Uid:         entry.Attributes.Uid,
+		Gid:         entry.Attributes.Gid,
+		UserName:    entry.Attributes.UserName,
+		Collection:  entry.Attributes.Collection,
+		Crtime:      entry.Attributes.Crtime,
+		Mtime:       entry.Attributes.Mtime,
+		Mime:        entry.Attributes.Mime,
+	}
+	return esEntry, id
+}
+
+func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) {
+	options := []elastic.ClientOptionFunc{}
+	options = append(options, elastic.SetURL(strings.Split(servers, ",")...))
+	options = append(options, elastic.SetSniff(false))
+	options = append(options, elastic.SetHealthcheck(false))
+	client, err := elastic.NewClient(options...)
+	if err != nil {
+		return nil, err
+	}
+	return func(resp *filer_pb.SubscribeMetadataResponse) error {
+		event := resp.EventNotification
+		if event.OldEntry != nil &&
+			(event.NewEntry == nil || resp.Directory != event.NewParentPath || event.OldEntry.Name != event.NewEntry.Name) {
+			// delete or not update the same file
+			dir, name := resp.Directory, event.OldEntry.Name
+			id := util.Md5String([]byte(util.NewFullPath(dir, name)))
+			println("delete", id)
+			_, err := client.Delete().Index(esIndex).Id(id).Do(context.Background())
+			return err
+		}
+		if event.NewEntry != nil {
+			// add a new file or update the same file
+			esEntry, id := toEsEntry(event)
+			value, err := jsoniter.Marshal(esEntry)
+			if err != nil {
+				return err
+			}
+			println(string(value))
+			_, err = client.Index().Index(esIndex).Id(id).BodyJson(string(value)).Do(context.Background())
+			return err
+		}
+		return nil
+	}, nil
+}

weed/command/filer_meta_tail_non_elastic.go

@@ -0,0 +1,14 @@
+//go:build !elastic
+// +build !elastic
+
+package command
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+)
+
+func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) {
+	return func(resp *filer_pb.SubscribeMetadataResponse) error {
+		return nil
+	}, nil
+}

weed/command/filer_sync.go

@@ -267,7 +267,10 @@ func genProcessFunction(sourcePath string, targetPath string, dataSink sink.Repl
 				return nil
 			}
 			key := buildKey(dataSink, message, targetPath, sourceOldKey, sourcePath)
-			return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
+			if !dataSink.IsIncremental() {
+				return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
+			}
+			return nil
 		}
 
 		// handle new entries

weed/command/iam.go

@@ -22,6 +22,7 @@ var (
 type IamOptions struct {
 	filer   *string
 	masters *string
+	ip      *string
 	port    *int
 }
 
@@ -29,6 +30,7 @@ func init() {
 	cmdIam.Run = runIam // break init cycle
 	iamStandaloneOptions.filer = cmdIam.Flag.String("filer", "localhost:8888", "filer server address")
 	iamStandaloneOptions.masters = cmdIam.Flag.String("master", "localhost:9333", "comma-separated master servers")
+	iamStandaloneOptions.ip = cmdIam.Flag.String("ip", util.DetectedHostAddress(), "iam server http listen ip address")
 	iamStandaloneOptions.port = cmdIam.Flag.Int("port", 8111, "iam server http listen port")
 }
 
@@ -65,7 +67,7 @@ func (iamopt *IamOptions) startIamServer() bool {
 		}
 	}
 
-	masters := pb.ServerAddresses(*iamopt.masters).ToAddresses()
+	masters := pb.ServerAddresses(*iamopt.masters).ToAddressMap()
 	router := mux.NewRouter().SkipClean(true)
 	_, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{
 		Masters:        masters,
@@ -81,12 +83,19 @@ func (iamopt *IamOptions) startIamServer() bool {
 	httpS := &http.Server{Handler: router}
 
 	listenAddress := fmt.Sprintf(":%d", *iamopt.port)
-	iamApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
+	iamApiListener, iamApiLocalListener, err := util.NewIpAndLocalListeners(*iamopt.ip, *iamopt.port, time.Duration(10)*time.Second)
 	if err != nil {
 		glog.Fatalf("IAM API Server listener on %s error: %v", listenAddress, err)
 	}
 
 	glog.V(0).Infof("Start Seaweed IAM API Server %s at http port %d", util.Version(), *iamopt.port)
+	if iamApiLocalListener != nil {
+		go func() {
+			if err = httpS.Serve(iamApiLocalListener); err != nil {
+				glog.Errorf("IAM API Server Fail to serve: %v", err)
+			}
+		}()
+	}
 	if err = httpS.Serve(iamApiListener); err != nil {
 		glog.Fatalf("IAM API Server Fail to serve: %v", err)
 	}

weed/command/imports.go

@@ -15,6 +15,7 @@ import (
 	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink"
 	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink"
 
+	_ "github.com/chrislusf/seaweedfs/weed/filer/arangodb"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/cassandra"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/etcd"
@@ -31,4 +32,5 @@ import (
 	_ "github.com/chrislusf/seaweedfs/weed/filer/redis2"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/redis3"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/sqlite"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/ydb"
 )

weed/command/master.go

@@ -1,15 +1,17 @@
 package command
 
 import (
+	"golang.org/x/exp/slices"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
 	"github.com/chrislusf/raft/protobuf"
 	stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
 	"github.com/gorilla/mux"
+	"github.com/spf13/viper"
 	"google.golang.org/grpc/reflection"
-	"net/http"
-	"os"
-	"sort"
-	"strings"
-	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/util/grace"
 
@@ -17,7 +19,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/server"
+	weed_server "github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
@@ -46,6 +48,8 @@ type MasterOptions struct {
 	metricsHttpPort    *int
 	heartbeatInterval  *time.Duration
 	electionTimeout    *time.Duration
+	raftHashicorp      *bool
+	raftBootstrap      *bool
 }
 
 func init() {
@@ -69,6 +73,8 @@ func init() {
 	m.raftResumeState = cmdMaster.Flag.Bool("resumeState", false, "resume previous state on start master server")
 	m.heartbeatInterval = cmdMaster.Flag.Duration("heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
 	m.electionTimeout = cmdMaster.Flag.Duration("electionTimeout", 10*time.Second, "election timeout of master servers")
+	m.raftHashicorp = cmdMaster.Flag.Bool("raftHashicorp", false, "use hashicorp raft")
+	m.raftBootstrap = cmdMaster.Flag.Bool("raftBootstrap", false, "Whether to bootstrap the Raft cluster")
 }
 
 var cmdMaster = &Command{
@@ -125,63 +131,122 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		*masterOption.portGrpc = 10000 + *masterOption.port
 	}
 	if *masterOption.ipBind == "" {
-		*masterOption.ipBind = "localhost"
+		*masterOption.ipBind = *masterOption.ip
 	}
 
 	myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.portGrpc, *masterOption.peers)
 
+	masterPeers := make(map[string]pb.ServerAddress)
+	for _, peer := range peers {
+		masterPeers[string(peer)] = peer
+	}
+
 	r := mux.NewRouter()
-	ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), peers)
+	ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), masterPeers)
 	listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port)
 	glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
-	masterListener, e := util.NewListener(listeningAddress, 0)
+	masterListener, masterLocalListner, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0)
 	if e != nil {
 		glog.Fatalf("Master startup error: %v", e)
 	}
+
 	// start raftServer
 	raftServerOption := &weed_server.RaftServerOption{
 		GrpcDialOption:    security.LoadClientTLS(util.GetViper(), "grpc.master"),
-		Peers:             peers,
+		Peers:             masterPeers,
 		ServerAddr:        myMasterAddress,
 		DataDir:           util.ResolvePath(*masterOption.metaFolder),
 		Topo:              ms.Topo,
 		RaftResumeState:   *masterOption.raftResumeState,
 		HeartbeatInterval: *masterOption.heartbeatInterval,
 		ElectionTimeout:   *masterOption.electionTimeout,
+		RaftBootstrap:     *m.raftBootstrap,
 	}
-	raftServer, err := weed_server.NewRaftServer(raftServerOption)
-	if raftServer == nil {
-		glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
+	var raftServer *weed_server.RaftServer
+	var err error
+	if *m.raftHashicorp {
+		if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil {
+			glog.Fatalf("NewHashicorpRaftServer: %s", err)
+		}
+	} else {
+		raftServer, err = weed_server.NewRaftServer(raftServerOption)
+		if raftServer == nil {
+			glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
+		}
 	}
 	ms.SetRaftServer(raftServer)
 	r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
+	if *m.raftHashicorp {
+		r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
+	}
 	// starting grpc server
 	grpcPort := *masterOption.portGrpc
-	grpcL, err := util.NewListener(util.JoinHostPort(*masterOption.ipBind, grpcPort), 0)
+	grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOption.ipBind, grpcPort, 0)
 	if err != nil {
 		glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
 	}
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
 	master_pb.RegisterSeaweedServer(grpcS, ms)
-	protobuf.RegisterRaftServer(grpcS, raftServer)
+	if *m.raftHashicorp {
+		raftServer.TransportManager.Register(grpcS)
+	} else {
+		protobuf.RegisterRaftServer(grpcS, raftServer)
+	}
 	reflection.Register(grpcS)
 	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOption.ipBind, grpcPort)
+	if grpcLocalL != nil {
+		go grpcS.Serve(grpcLocalL)
+	}
 	go grpcS.Serve(grpcL)
 
-	go func() {
-		time.Sleep(1500 * time.Millisecond)
-		if ms.Topo.RaftServer.Leader() == "" && ms.Topo.RaftServer.IsLogEmpty() && isTheFirstOne(myMasterAddress, peers) {
-			if ms.MasterClient.FindLeaderFromOtherPeers(myMasterAddress) == "" {
-				raftServer.DoJoinCommand()
+	timeSleep := 1500 * time.Millisecond
+	if !*m.raftHashicorp {
+		go func() {
+			time.Sleep(timeSleep)
+			if ms.Topo.RaftServer.Leader() == "" && ms.Topo.RaftServer.IsLogEmpty() && isTheFirstOne(myMasterAddress, peers) {
+				if ms.MasterClient.FindLeaderFromOtherPeers(myMasterAddress) == "" {
+					raftServer.DoJoinCommand()
+				}
 			}
-		}
-	}()
+		}()
+	}
 
 	go ms.MasterClient.KeepConnectedToMaster()
 
 	// start http server
+	var (
+		clientCertFile,
+		certFile,
+		keyFile string
+	)
+	useTLS := false
+	useMTLS := false
+
+	if viper.GetString("https.master.key") != "" {
+		useTLS = true
+		certFile = viper.GetString("https.master.cert")
+		keyFile = viper.GetString("https.master.key")
+	}
+
+	if viper.GetString("https.master.ca") != "" {
+		useMTLS = true
+		clientCertFile = viper.GetString("https.master.ca")
+	}
+
 	httpS := &http.Server{Handler: r}
-	go httpS.Serve(masterListener)
+	if masterLocalListner != nil {
+		go httpS.Serve(masterLocalListner)
+	}
+
+	if useMTLS {
+		httpS.TLSConfig = security.LoadClientTLSHTTP(clientCertFile)
+	}
+
+	if useTLS {
+		go httpS.ServeTLS(masterListener, certFile, keyFile)
+	} else {
+		go httpS.Serve(masterListener)
+	}
 
 	select {}
 }
@@ -209,8 +274,8 @@ func checkPeers(masterIp string, masterPort int, masterGrpcPort int, peers strin
 }
 
 func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool {
-	sort.Slice(peers, func(i, j int) bool {
-		return strings.Compare(string(peers[i]), string(peers[j])) < 0
+	slices.SortFunc(peers, func(a, b pb.ServerAddress) bool {
+		return strings.Compare(string(a), string(b)) < 0
 	})
 	if len(peers) <= 0 {
 		return true

weed/command/master_follower.go

@@ -3,17 +3,18 @@ package command
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"time"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/server"
+	weed_server "github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
 	"google.golang.org/grpc/reflection"
-	"net/http"
-	"time"
 )
 
 var (
@@ -45,13 +46,13 @@ var cmdMasterFollower = &Command{
 	Short:     "start a master follower",
 	Long: `start a master follower to provide volume=>location mapping service
 
-	The master follower does not participate in master election. 
+	The master follower does not participate in master election.
 	It just follow the existing masters, and listen for any volume location changes.
 
 	In most cases, the master follower is not needed. In big data centers with thousands of volume
 	servers. In theory, the master may have trouble to keep up with the write requests and read requests.
 
-	The master follower can relieve the master from from read requests, which only needs to 
+	The master follower can relieve the master from from read requests, which only needs to
 	lookup a fileId or volumeId.
 
 	The master follower currently can handle fileId lookup requests:
@@ -82,7 +83,7 @@ func runMasterFollower(cmd *Command, args []string) bool {
 func startMasterFollower(masterOptions MasterOptions) {
 
 	// collect settings from main masters
-	masters := pb.ServerAddresses(*mf.peers).ToAddresses()
+	masters := pb.ServerAddresses(*mf.peers).ToAddressMap()
 
 	var err error
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.master")
@@ -112,21 +113,21 @@ func startMasterFollower(masterOptions MasterOptions) {
 	option.IsFollower = true
 
 	if *masterOptions.ipBind == "" {
-		*masterOptions.ipBind = "localhost"
+		*masterOptions.ipBind = *masterOptions.ip
 	}
 
 	r := mux.NewRouter()
 	ms := weed_server.NewMasterServer(r, option, masters)
 	listeningAddress := util.JoinHostPort(*masterOptions.ipBind, *masterOptions.port)
 	glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
-	masterListener, e := util.NewListener(listeningAddress, 0)
+	masterListener, masterLocalListner, e := util.NewIpAndLocalListeners(*masterOptions.ipBind, *masterOptions.port, 0)
 	if e != nil {
 		glog.Fatalf("Master startup error: %v", e)
 	}
 
 	// starting grpc server
 	grpcPort := *masterOptions.portGrpc
-	grpcL, err := util.NewListener(util.JoinHostPort(*masterOptions.ipBind, grpcPort), 0)
+	grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOptions.ipBind, grpcPort, 0)
 	if err != nil {
 		glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
 	}
@@ -134,12 +135,18 @@ func startMasterFollower(masterOptions MasterOptions) {
 	master_pb.RegisterSeaweedServer(grpcS, ms)
 	reflection.Register(grpcS)
 	glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOptions.ip, grpcPort)
+	if grpcLocalL != nil {
+		go grpcS.Serve(grpcLocalL)
+	}
 	go grpcS.Serve(grpcL)
 
 	go ms.MasterClient.KeepConnectedToMaster()
 
 	// start http server
 	httpS := &http.Server{Handler: r}
+	if masterLocalListner != nil {
+		go httpS.Serve(masterLocalListner)
+	}
 	go httpS.Serve(masterListener)
 
 	select {}

weed/command/mount.go

@@ -29,6 +29,7 @@ type MountOptions struct {
 	readOnly           *bool
 	debug              *bool
 	debugPort          *int
+	localSocket        *string
 }
 
 var (
@@ -63,6 +64,7 @@ func init() {
 	mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only")
 	mountOptions.debug = cmdMount.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
 	mountOptions.debugPort = cmdMount.Flag.Int("debug.port", 6061, "http port for debugging")
+	mountOptions.localSocket = cmdMount.Flag.String("localSocket", "", "default to /tmp/seaweedfs-mount-<mount_dir_hash>.sock")
 
 	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
 	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")

weed/command/mount_std.go

@@ -12,9 +12,12 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/mount/unmount"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/hanwen/go-fuse/v2/fuse"
+	"google.golang.org/grpc/reflection"
+	"net"
 	"net/http"
 	"os"
 	"os/user"
@@ -98,6 +101,22 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 
 	unmount.Unmount(dir)
 
+	// start on local unix socket
+	if *option.localSocket == "" {
+		mountDirHash := util.HashToInt32([]byte(dir))
+		if mountDirHash < 0 {
+			mountDirHash = -mountDirHash
+		}
+		*option.localSocket = fmt.Sprintf("/tmp/seaweefs-mount-%d.sock", mountDirHash)
+	}
+	if err := os.Remove(*option.localSocket); err != nil && !os.IsNotExist(err) {
+		glog.Fatalf("Failed to remove %s, error: %s", *option.localSocket, err.Error())
+	}
+	montSocketListener, err := net.Listen("unix", *option.localSocket)
+	if err != nil {
+		glog.Fatalf("Failed to listen on %s: %v", *option.localSocket, err)
+	}
+
 	// detect mount folder mode
 	if *option.dirAutoCreate {
 		os.MkdirAll(dir, os.FileMode(0777)&^umask)
@@ -229,6 +248,11 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		unmount.Unmount(dir)
 	})
 
+	grpcS := pb.NewGrpcServer()
+	mount_pb.RegisterSeaweedMountServer(grpcS, seaweedFileSystem)
+	reflection.Register(grpcS)
+	go grpcS.Serve(montSocketListener)
+
 	seaweedFileSystem.StartBackgroundTasks()
 
 	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH)

weed/command/msg_broker.go

@@ -95,7 +95,7 @@ func (msgBrokerOpt *MessageBrokerOptions) startQueueServer() bool {
 	}, grpcDialOption)
 
 	// start grpc listener
-	grpcL, err := util.NewListener(util.JoinHostPort("", *msgBrokerOpt.port), 0)
+	grpcL, _, err := util.NewIpAndLocalListeners("", *msgBrokerOpt.port, 0)
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", *msgBrokerOpt.port, err)
 	}

weed/command/s3.go

@@ -24,17 +24,18 @@ var (
 )
 
 type S3Options struct {
-	filer            *string
-	bindIp           *string
-	port             *int
-	config           *string
-	domainName       *string
-	tlsPrivateKey    *string
-	tlsCertificate   *string
-	metricsHttpPort  *int
-	allowEmptyFolder *bool
-	auditLogConfig   *string
-	localFilerSocket *string
+	filer                     *string
+	bindIp                    *string
+	port                      *int
+	config                    *string
+	domainName                *string
+	tlsPrivateKey             *string
+	tlsCertificate            *string
+	metricsHttpPort           *int
+	allowEmptyFolder          *bool
+	allowDeleteBucketNotEmpty *bool
+	auditLogConfig            *string
+	localFilerSocket          *string
 }
 
 func init() {
@@ -49,6 +50,7 @@ func init() {
 	s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
 	s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", true, "allow empty folders")
+	s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 }
 
 var cmdS3 = &Command{
@@ -178,14 +180,15 @@ func (s3opt *S3Options) startS3Server() bool {
 	router := mux.NewRouter().SkipClean(true)
 
 	_, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
-		Filer:            filerAddress,
-		Port:             *s3opt.port,
-		Config:           *s3opt.config,
-		DomainName:       *s3opt.domainName,
-		BucketsPath:      filerBucketsPath,
-		GrpcDialOption:   grpcDialOption,
-		AllowEmptyFolder: *s3opt.allowEmptyFolder,
-		LocalFilerSocket: s3opt.localFilerSocket,
+		Filer:                     filerAddress,
+		Port:                      *s3opt.port,
+		Config:                    *s3opt.config,
+		DomainName:                *s3opt.domainName,
+		BucketsPath:               filerBucketsPath,
+		GrpcDialOption:            grpcDialOption,
+		AllowEmptyFolder:          *s3opt.allowEmptyFolder,
+		AllowDeleteBucketNotEmpty: *s3opt.allowDeleteBucketNotEmpty,
+		LocalFilerSocket:          s3opt.localFilerSocket,
 	})
 	if s3ApiServer_err != nil {
 		glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
@@ -198,7 +201,7 @@ func (s3opt *S3Options) startS3Server() bool {
 	}
 
 	listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.port)
-	s3ApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
+	s3ApiListener, s3ApiLocalListner, err := util.NewIpAndLocalListeners(*s3opt.bindIp, *s3opt.port, time.Duration(10)*time.Second)
 	if err != nil {
 		glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
 	}
@@ -212,11 +215,25 @@ func (s3opt *S3Options) startS3Server() bool {
 
 	if *s3opt.tlsPrivateKey != "" {
 		glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
+		if s3ApiLocalListner != nil {
+			go func() {
+				if err = httpS.ServeTLS(s3ApiLocalListner, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
+					glog.Fatalf("S3 API Server Fail to serve: %v", err)
+				}
+			}()
+		}
 		if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
 			glog.Fatalf("S3 API Server Fail to serve: %v", err)
 		}
 	} else {
 		glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.Version(), *s3opt.port)
+		if s3ApiLocalListner != nil {
+			go func() {
+				if err = httpS.Serve(s3ApiLocalListner); err != nil {
+					glog.Fatalf("S3 API Server Fail to serve: %v", err)
+				}
+			}()
+		}
 		if err = httpS.Serve(s3ApiListener); err != nil {
 			glog.Fatalf("S3 API Server Fail to serve: %v", err)
 		}

weed/command/scaffold/filer.toml

@@ -285,6 +285,29 @@ healthcheck_enabled = false
 index.max_result_window = 10000
 
 
+[arangodb] # in development dont use it
+enabled = false
+db_name = "seaweedfs"
+servers=["http://localhost:8529"] # list of servers to connect to
+# only basic auth supported for now
+username=""
+password=""
+# skip tls cert validation
+insecure_skip_verify = true
+
+[ydb] # https://ydb.tech/
+enabled = false
+dsn = "grpc://localhost:2136?database=/local"
+prefix = "seaweedfs"
+useBucketPrefix = true # Fast Bucket Deletion
+poolSizeLimit = 50
+dialTimeOut = 10
+
+# Authenticate produced with one of next environment variables:
+# YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=<path/to/sa_key_file> — used service account key file by path
+# YDB_ANONYMOUS_CREDENTIALS="1" — used for authenticate with anonymous access. Anonymous access needs for connect to testing YDB installation
+# YDB_METADATA_CREDENTIALS="1" — used metadata service for authenticate to YDB from yandex cloud virtual machine or from yandex function
+# YDB_ACCESS_TOKEN_CREDENTIALS=<access_token> — used for authenticate to YDB with short-life access token. For example, access token may be IAM token
 
 ##########################
 ##########################

weed/command/scaffold/master.toml

@@ -36,6 +36,7 @@ aws_secret_access_key = ""     # if empty, loads from the shared credentials fil
 region = "us-east-2"
 bucket = "your_bucket_name"    # an existing bucket
 endpoint = ""
+storage_class = "STANDARD_IA"
 
 # create this number of logical volumes if no more writable volumes
 # count_x means how many copies of data.

weed/command/scaffold/security.toml

@@ -83,7 +83,13 @@ key = ""
 #     this does not work with other clients, e.g., "weed filer|mount" etc, yet.
 [https.client]
 enabled = true
+
 [https.volume]
 cert = ""
 key = ""
+ca = ""
 
+[https.master]
+cert = ""
+key = ""
+ca = ""

weed/command/server.go

@@ -101,6 +101,7 @@ func init() {
 	masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
 	masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
 
+	filerOptions.filerGroup = cmdServer.Flag.String("filer.filerGroup", "", "share metadata with other filers in the same filerGroup")
 	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
 	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
 	filerOptions.portGrpc = cmdServer.Flag.Int("filer.port.grpc", 0, "filer server grpc listen port")
@@ -138,6 +139,7 @@ func init() {
 	s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
 	s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
 	s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
+	s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
 
 	iamOptions.port = cmdServer.Flag.Int("iam.port", 8111, "iam server http listen port")
 
@@ -191,10 +193,11 @@ func runServer(cmd *Command, args []string) bool {
 	// ip address
 	masterOptions.ip = serverIp
 	masterOptions.ipBind = serverBindIp
-	filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
+	filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddressMap()
 	filerOptions.ip = serverIp
 	filerOptions.bindIp = serverBindIp
 	s3Options.bindIp = serverBindIp
+	iamOptions.ip = serverBindIp
 	iamOptions.masters = masterOptions.peers
 	serverOptions.v.ip = serverIp
 	serverOptions.v.bindIp = serverBindIp

weed/command/shell.go

@@ -18,6 +18,7 @@ var (
 func init() {
 	cmdShell.Run = runShell // break init cycle
 	shellOptions.Masters = cmdShell.Flag.String("master", "", "comma-separated master servers, e.g. localhost:9333")
+	shellOptions.FilerGroup = cmdShell.Flag.String("filerGroup", "", "filerGroup for the filers")
 	shellInitialFiler = cmdShell.Flag.String("filer", "", "filer host and port, e.g. localhost:8888")
 	shellCluster = cmdShell.Flag.String("cluster", "", "cluster defined in shell.toml")
 }

weed/command/volume.go

@@ -2,7 +2,6 @@ package command
 
 import (
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/storage/types"
 	"net/http"
 	httppprof "net/http/pprof"
 	"os"
@@ -11,6 +10,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
+
 	"github.com/spf13/viper"
 	"google.golang.org/grpc"
 
@@ -24,7 +25,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
-	"github.com/chrislusf/seaweedfs/weed/server"
+	weed_server "github.com/chrislusf/seaweedfs/weed/server"
 	stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -371,7 +372,14 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpd
 		StopTimeout: 30 * time.Second,
 		CertFile:    certFile,
 		KeyFile:     keyFile}
-	clusterHttpServer := httpDown.Serve(&http.Server{Handler: handler}, listener)
+	httpS := &http.Server{Handler: handler}
+
+	if viper.GetString("https.volume.ca") != "" {
+		clientCertFile := viper.GetString("https.volume.ca")
+		httpS.TLSConfig = security.LoadClientTLSHTTP(clientCertFile)
+	}
+
+	clusterHttpServer := httpDown.Serve(httpS, listener)
 	go func() {
 		if e := clusterHttpServer.Wait(); e != nil {
 			glog.Fatalf("Volume server fail to serve: %v", e)

weed/filer/abstract_sql/abstract_sql_store.go

@@ -156,7 +156,7 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent
 		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
 	}
 
-	if len(entry.Chunks) > 50 {
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
 		meta = util.MaybeGzipData(meta)
 	}
 

weed/filer/abstract_sql/abstract_sql_store_kv.go

@@ -18,7 +18,7 @@ func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []by
 		return fmt.Errorf("findDB: %v", err)
 	}
 
-	dirStr, dirHash, name := genDirAndName(key)
+	dirStr, dirHash, name := GenDirAndName(key)
 
 	res, err := db.ExecContext(ctx, store.GetSqlInsert(DEFAULT_TABLE), dirHash, name, dirStr, value)
 	if err == nil {
@@ -53,7 +53,7 @@ func (store *AbstractSqlStore) KvGet(ctx context.Context, key []byte) (value []b
 		return nil, fmt.Errorf("findDB: %v", err)
 	}
 
-	dirStr, dirHash, name := genDirAndName(key)
+	dirStr, dirHash, name := GenDirAndName(key)
 	row := db.QueryRowContext(ctx, store.GetSqlFind(DEFAULT_TABLE), dirHash, name, dirStr)
 
 	err = row.Scan(&value)
@@ -76,7 +76,7 @@ func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err er
 		return fmt.Errorf("findDB: %v", err)
 	}
 
-	dirStr, dirHash, name := genDirAndName(key)
+	dirStr, dirHash, name := GenDirAndName(key)
 
 	res, err := db.ExecContext(ctx, store.GetSqlDelete(DEFAULT_TABLE), dirHash, name, dirStr)
 	if err != nil {
@@ -92,7 +92,7 @@ func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err er
 
 }
 
-func genDirAndName(key []byte) (dirStr string, dirHash int64, name string) {
+func GenDirAndName(key []byte) (dirStr string, dirHash int64, name string) {
 	for len(key) < 8 {
 		key = append(key, 0)
 	}

weed/filer/arangodb/arangodb_store.go

@@ -0,0 +1,347 @@
+package arangodb
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/arangodb/go-driver"
+	"github.com/arangodb/go-driver/http"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &ArangodbStore{})
+}
+
+var (
+	BUCKET_PREFIX      = "/buckets"
+	DEFAULT_COLLECTION = "seaweed_no_bucket"
+	KVMETA_COLLECTION  = "seaweed_kvmeta"
+)
+
+type ArangodbStore struct {
+	connect      driver.Connection
+	client       driver.Client
+	database     driver.Database
+	kvCollection driver.Collection
+
+	buckets map[string]driver.Collection
+	mu      sync.RWMutex
+
+	databaseName string
+}
+
+type Model struct {
+	Key       string `json:"_key"`
+	Directory string `json:"directory,omitempty"`
+	Name      string `json:"name,omitempty"`
+	Ttl       string `json:"ttl,omitempty"`
+
+	//arangodb does not support binary blobs
+	//we encode byte slice into uint64 slice
+	//see helpers.go
+	Meta []uint64 `json:"meta"`
+}
+
+func (store *ArangodbStore) GetName() string {
+	return "arangodb"
+}
+
+func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	store.buckets = make(map[string]driver.Collection, 3)
+	store.databaseName = configuration.GetString(prefix + "db_name")
+	return store.connection(configuration.GetStringSlice(prefix+"servers"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetBool(prefix+"insecure_skip_verify"),
+	)
+}
+
+func (store *ArangodbStore) connection(uris []string, user string, pass string, insecure bool) (err error) {
+	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
+
+	store.connect, err = http.NewConnection(http.ConnectionConfig{
+		Endpoints: uris,
+		TLSConfig: &tls.Config{
+			InsecureSkipVerify: insecure,
+		},
+	})
+	if err != nil {
+		return err
+	}
+	store.client, err = driver.NewClient(driver.ClientConfig{
+		Connection:     store.connect,
+		Authentication: driver.BasicAuthentication(user, pass),
+	})
+	if err != nil {
+		return err
+	}
+	ok, err := store.client.DatabaseExists(ctx, store.databaseName)
+	if err != nil {
+		return err
+	}
+	if ok {
+		store.database, err = store.client.Database(ctx, store.databaseName)
+	} else {
+		store.database, err = store.client.CreateDatabase(ctx, store.databaseName, &driver.CreateDatabaseOptions{})
+	}
+	if err != nil {
+		return err
+	}
+	if store.kvCollection, err = store.ensureCollection(ctx, KVMETA_COLLECTION); err != nil {
+		return err
+	}
+	return err
+}
+
+type key int
+
+const (
+	transactionKey key = 0
+)
+
+func (store *ArangodbStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	keys := make([]string, 0, len(store.buckets)+1)
+	for k := range store.buckets {
+		keys = append(keys, k)
+	}
+	keys = append(keys, store.kvCollection.Name())
+	txn, err := store.database.BeginTransaction(ctx, driver.TransactionCollections{
+		Exclusive: keys,
+	}, &driver.BeginTransactionOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return context.WithValue(ctx, transactionKey, txn), nil
+}
+
+func (store *ArangodbStore) CommitTransaction(ctx context.Context) error {
+	val := ctx.Value(transactionKey)
+	cast, ok := val.(driver.TransactionID)
+	if !ok {
+		return fmt.Errorf("txn cast fail %s:", val)
+	}
+	err := store.database.CommitTransaction(ctx, cast, &driver.CommitTransactionOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (store *ArangodbStore) RollbackTransaction(ctx context.Context) error {
+	val := ctx.Value(transactionKey)
+	cast, ok := val.(driver.TransactionID)
+	if !ok {
+		return fmt.Errorf("txn cast fail %s:", val)
+	}
+	err := store.database.AbortTransaction(ctx, cast, &driver.AbortTransactionOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	dir, name := entry.FullPath.DirAndName()
+	meta, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
+	}
+
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
+		meta = util.MaybeGzipData(meta)
+	}
+	model := &Model{
+		Key:       hashString(string(entry.FullPath)),
+		Directory: dir,
+		Name:      name,
+		Meta:      bytesToArray(meta),
+	}
+	if entry.TtlSec > 0 {
+		model.Ttl = time.Now().Add(time.Second * time.Duration(entry.TtlSec)).Format(time.RFC3339)
+	} else {
+		model.Ttl = ""
+	}
+
+	targetCollection, err := store.extractBucketCollection(ctx, entry.FullPath)
+	if err != nil {
+		return err
+	}
+	_, err = targetCollection.CreateDocument(ctx, model)
+	if driver.IsConflict(err) {
+		return store.UpdateEntry(ctx, entry)
+	}
+
+	if err != nil {
+		return fmt.Errorf("InsertEntry %s: %v", entry.FullPath, err)
+	}
+
+	return nil
+
+}
+
+func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	dir, name := entry.FullPath.DirAndName()
+	meta, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
+	}
+
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
+		meta = util.MaybeGzipData(meta)
+	}
+	model := &Model{
+		Key:       hashString(string(entry.FullPath)),
+		Directory: dir,
+		Name:      name,
+		Meta:      bytesToArray(meta),
+	}
+	if entry.TtlSec > 0 {
+		model.Ttl = time.Now().Add(time.Duration(entry.TtlSec) * time.Second).Format(time.RFC3339)
+	} else {
+		model.Ttl = "none"
+	}
+	targetCollection, err := store.extractBucketCollection(ctx, entry.FullPath)
+	if err != nil {
+		return err
+	}
+	_, err = targetCollection.UpdateDocument(ctx, model.Key, model)
+	if err != nil {
+		return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err)
+	}
+
+	return nil
+}
+
+func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+	var data Model
+	targetCollection, err := store.extractBucketCollection(ctx, fullpath)
+	if err != nil {
+		return nil, err
+	}
+	_, err = targetCollection.ReadDocument(ctx, hashString(string(fullpath)), &data)
+	if err != nil {
+		if driver.IsNotFound(err) {
+			return nil, filer_pb.ErrNotFound
+		}
+		glog.Errorf("find %s: %v", fullpath, err)
+		return nil, filer_pb.ErrNotFound
+	}
+	if len(data.Meta) == 0 {
+		return nil, filer_pb.ErrNotFound
+	}
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(arrayToBytes(data.Meta)))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
+	targetCollection, err := store.extractBucketCollection(ctx, fullpath)
+	if err != nil {
+		return err
+	}
+	_, err = targetCollection.RemoveDocument(ctx, hashString(string(fullpath)))
+	if err != nil && !driver.IsNotFound(err) {
+		glog.Errorf("find %s: %v", fullpath, err)
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+	return nil
+}
+
+// this runs in log time
+func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
+	var query string
+	targetCollection, err := store.extractBucketCollection(ctx, fullpath)
+	if err != nil {
+		return err
+	}
+	query = query + fmt.Sprintf(`
+	for d in %s
+	filter starts_with(d.directory, "%s/")  || d.directory == "%s"
+	remove d._key in %s`,
+		targetCollection.Name(),
+		strings.Join(strings.Split(string(fullpath), "/"), ","),
+		string(fullpath),
+		targetCollection.Name(),
+	)
+	cur, err := store.database.Query(ctx, query, nil)
+	if err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+	defer cur.Close()
+	return nil
+}
+
+func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
+}
+
+func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	targetCollection, err := store.extractBucketCollection(ctx, dirPath+"/")
+	if err != nil {
+		return lastFileName, err
+	}
+	query := "for d in " + targetCollection.Name()
+	if includeStartFile {
+		query = query + " filter d.name >= \"" + startFileName + "\" "
+	} else {
+		query = query + " filter d.name > \"" + startFileName + "\" "
+	}
+	if prefix != "" {
+		query = query + fmt.Sprintf(`&& starts_with(d.name, "%s")`, prefix)
+	}
+	query = query + `
+filter d.directory == @dir
+sort d.name asc
+`
+	if limit > 0 {
+		query = query + "limit " + strconv.Itoa(int(limit))
+	}
+	query = query + "\n return d"
+	cur, err := store.database.Query(ctx, query, map[string]interface{}{"dir": dirPath})
+	if err != nil {
+		return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err)
+	}
+	defer cur.Close()
+	for cur.HasMore() {
+		var data Model
+		_, err = cur.ReadDocument(ctx, &data)
+		if err != nil {
+			break
+		}
+		entry := &filer.Entry{
+			FullPath: util.NewFullPath(data.Directory, data.Name),
+		}
+		lastFileName = data.Name
+		converted := arrayToBytes(data.Meta)
+		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil {
+			err = decodeErr
+			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
+			break
+		}
+
+		if !eachEntryFunc(entry) {
+			break
+		}
+
+	}
+	return lastFileName, err
+}
+
+func (store *ArangodbStore) Shutdown() {
+}

weed/filer/arangodb/arangodb_store_bucket.go

@@ -0,0 +1,40 @@
+package arangodb
+
+import (
+	"context"
+	"github.com/arangodb/go-driver"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+)
+
+var _ filer.BucketAware = (*ArangodbStore)(nil)
+
+func (store *ArangodbStore) OnBucketCreation(bucket string) {
+	timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	// create the collection && add to cache
+	_, err := store.ensureBucket(timeout, bucket)
+	if err != nil {
+		glog.Errorf("bucket create %s: %v", bucket, err)
+	}
+}
+func (store *ArangodbStore) OnBucketDeletion(bucket string) {
+	timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	collection, err := store.ensureBucket(timeout, bucket)
+	if err != nil {
+		glog.Errorf("bucket delete %s: %v", bucket, err)
+		return
+	}
+	err = collection.Remove(timeout)
+	if err != nil && !driver.IsNotFound(err) {
+		glog.Errorf("bucket delete %s: %v", bucket, err)
+		return
+	}
+}
+func (store *ArangodbStore) CanDropWholeBucket() bool {
+	return true
+}

weed/filer/arangodb/arangodb_store_kv.go

@@ -0,0 +1,54 @@
+package arangodb
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/arangodb/go-driver"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+)
+
+func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+	model := &Model{
+		Key:       hashString(".kvstore." + string(key)),
+		Directory: ".kvstore." + string(key),
+		Meta:      bytesToArray(value),
+	}
+
+	exists, err := store.kvCollection.DocumentExists(ctx, model.Key)
+	if err != nil {
+		return fmt.Errorf("kv put: %v", err)
+	}
+	if exists {
+		_, err = store.kvCollection.UpdateDocument(ctx, model.Key, model)
+	} else {
+		_, err = store.kvCollection.CreateDocument(ctx, model)
+	}
+	if err != nil {
+		return fmt.Errorf("kv put: %v", err)
+	}
+
+	return nil
+}
+func (store *ArangodbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+	var model Model
+	_, err = store.kvCollection.ReadDocument(ctx, hashString(".kvstore."+string(key)), &model)
+	if driver.IsNotFound(err) {
+		return nil, filer.ErrKvNotFound
+	}
+	if err != nil {
+		glog.Errorf("kv get: %s %v", string(key), err)
+		return nil, filer.ErrKvNotFound
+	}
+	return arrayToBytes(model.Meta), nil
+}
+
+func (store *ArangodbStore) KvDelete(ctx context.Context, key []byte) (err error) {
+	_, err = store.kvCollection.RemoveDocument(ctx, hashString(".kvstore."+string(key)))
+	if err != nil {
+		glog.Errorf("kv del: %v", err)
+		return filer.ErrKvNotFound
+	}
+	return nil
+}

weed/filer/arangodb/helpers.go

@@ -0,0 +1,136 @@
+package arangodb
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/binary"
+	"encoding/hex"
+	"io"
+	"strings"
+
+	"github.com/arangodb/go-driver"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+//convert a string into arango-key safe hex bytes hash
+func hashString(dir string) string {
+	h := md5.New()
+	io.WriteString(h, dir)
+	b := h.Sum(nil)
+	return hex.EncodeToString(b)
+}
+
+// convert slice of bytes into slice of uint64
+// the first uint64 indicates the length in bytes
+func bytesToArray(bs []byte) []uint64 {
+	out := make([]uint64, 0, 2+len(bs)/8)
+	out = append(out, uint64(len(bs)))
+	for len(bs)%8 != 0 {
+		bs = append(bs, 0)
+	}
+	for i := 0; i < len(bs); i = i + 8 {
+		out = append(out, binary.BigEndian.Uint64(bs[i:]))
+	}
+	return out
+}
+
+// convert from slice of uint64 back to bytes
+// if input length is 0 or 1, will return nil
+func arrayToBytes(xs []uint64) []byte {
+	if len(xs) < 2 {
+		return nil
+	}
+	first := xs[0]
+	out := make([]byte, len(xs)*8) // i think this can actually be len(xs)*8-8, but i dont think an extra 8 bytes hurts...
+	for i := 1; i < len(xs); i = i + 1 {
+		binary.BigEndian.PutUint64(out[((i-1)*8):], xs[i])
+	}
+	return out[:first]
+}
+
+// gets the collection the bucket points to from filepath
+func (store *ArangodbStore) extractBucketCollection(ctx context.Context, fullpath util.FullPath) (c driver.Collection, err error) {
+	bucket, _ := extractBucket(fullpath)
+	if bucket == "" {
+		bucket = DEFAULT_COLLECTION
+	}
+	c, err = store.ensureBucket(ctx, bucket)
+	if err != nil {
+		return nil, err
+	}
+	return c, err
+}
+
+// called by extractBucketCollection
+func extractBucket(fullpath util.FullPath) (string, string) {
+	if !strings.HasPrefix(string(fullpath), BUCKET_PREFIX+"/") {
+		return "", string(fullpath)
+	}
+	if strings.Count(string(fullpath), "/") < 3 {
+		return "", string(fullpath)
+	}
+	bucketAndObjectKey := string(fullpath)[len(BUCKET_PREFIX+"/"):]
+	t := strings.Index(bucketAndObjectKey, "/")
+	bucket := bucketAndObjectKey
+	shortPath := "/"
+	if t > 0 {
+		bucket = bucketAndObjectKey[:t]
+		shortPath = string(util.FullPath(bucketAndObjectKey[t:]))
+	}
+	return bucket, shortPath
+}
+
+// get bucket collection from cache. if not exist, creates the buckets collection and grab it
+func (store *ArangodbStore) ensureBucket(ctx context.Context, bucket string) (bc driver.Collection, err error) {
+	var ok bool
+	store.mu.RLock()
+	bc, ok = store.buckets[bucket]
+	store.mu.RUnlock()
+	if ok {
+		return bc, nil
+	}
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	store.buckets[bucket], err = store.ensureCollection(ctx, bucket)
+	if err != nil {
+		return nil, err
+	}
+	return store.buckets[bucket], nil
+}
+
+// creates collection if not exist, ensures indices if not exist
+func (store *ArangodbStore) ensureCollection(ctx context.Context, name string) (c driver.Collection, err error) {
+	ok, err := store.database.CollectionExists(ctx, name)
+	if err != nil {
+		return
+	}
+	if ok {
+		c, err = store.database.Collection(ctx, name)
+	} else {
+		c, err = store.database.CreateCollection(ctx, name, &driver.CreateCollectionOptions{})
+	}
+	if err != nil {
+		return
+	}
+	// ensure indices
+	if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory", "name"},
+		&driver.EnsurePersistentIndexOptions{
+			Name: "directory_name_multi", Unique: true,
+		}); err != nil {
+		return
+	}
+	if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory"},
+		&driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil {
+		return
+	}
+	if _, _, err = c.EnsureTTLIndex(ctx, "ttl", 1,
+		&driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil {
+		return
+	}
+	if _, _, err = c.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{
+		Name: "IDX_name",
+	}); err != nil {
+		return
+	}
+	return c, nil
+}

weed/filer/arangodb/readme.md

@@ -0,0 +1,52 @@
+##arangodb
+
+database: https://github.com/arangodb/arangodb
+go driver: https://github.com/arangodb/go-driver
+
+options:
+
+```
+[arangodb]
+enabled=true
+db_name="seaweedfs"
+servers=["http://localhost:8529"]
+#basic auth
+user="root"
+pass="test"
+
+# tls settings
+insecure_skip_verify=true
+```
+
+i test using this dev database:
+`docker run -p 8529:8529 -e ARANGO_ROOT_PASSWORD=test arangodb/arangodb:3.9.0`
+
+
+## features i don't personally need but are missing
+ [ ] provide tls cert to arango
+ [ ] authentication that is not basic auth
+ [ ] synchronise endpoint interval config
+ [ ] automatic creation of custom index
+ [ ] configure default arangodb collection sharding rules
+ [ ] configure default arangodb collection replication rules
+
+
+## complexity
+
+ok, so if https://www.arangodb.com/docs/stable/indexing-index-basics.html#persistent-index is correct
+
+O(1)
+- InsertEntry
+- UpdateEntry
+- FindEntry
+- DeleteEntry
+- KvPut
+- KvGet
+- KvDelete
+
+O(log(BUCKET_SIZE))
+- DeleteFolderChildren
+
+O(log(DIRECTORY_SIZE))
+- ListDirectoryEntries
+- ListDirectoryPrefixedEntries

weed/filer/cassandra/cassandra_store.go

@@ -100,7 +100,7 @@ func (store *CassandraStore) InsertEntry(ctx context.Context, entry *filer.Entry
 		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
 	}
 
-	if len(entry.Chunks) > 50 {
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
 		meta = util.MaybeGzipData(meta)
 	}
 

weed/filer/elastic/v7/doc.go

@@ -0,0 +1,9 @@
+/*
+
+Package elastic is for elastic filer store.
+
+The referenced "github.com/olivere/elastic/v7" library is too big when compiled.
+So this is only compiled in "make full_install".
+
+*/
+package elastic

weed/filer/elastic/v7/elastic_store.go

@@ -1,3 +1,6 @@
+//go:build elastic
+// +build elastic
+
 package elastic
 
 import (

weed/filer/elastic/v7/elastic_store_kv.go

@@ -1,3 +1,6 @@
+//go:build elastic
+// +build elastic
+
 package elastic
 
 import (

weed/filer/etcd/etcd_store.go

@@ -82,7 +82,7 @@ func (store *EtcdStore) InsertEntry(ctx context.Context, entry *filer.Entry) (er
 		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
 	}
 
-	if len(entry.Chunks) > 50 {
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
 		meta = weed_util.MaybeGzipData(meta)
 	}
 
@@ -152,7 +152,7 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u
 	}
 
 	resp, err := store.client.Get(ctx, string(lastFileStart),
-		clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortDescend))
+		clientv3.WithFromKey(), clientv3.WithLimit(limit+1))
 	if err != nil {
 		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
 	}

weed/filer/etcd/etcd_store_test.go

@@ -0,0 +1,16 @@
+package etcd
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer/store_test"
+	"testing"
+)
+
+func TestStore(t *testing.T) {
+	// run "make test_etcd" under docker folder.
+	// to set up local env
+	if false {
+		store := &EtcdStore{}
+		store.initialize("localhost:2379", "3s")
+		store_test.TestFilerStore(t, store)
+	}
+}

weed/filer/filechunks.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
+	"golang.org/x/exp/slices"
 	"math"
-	"sort"
 	"sync"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@@ -23,7 +23,16 @@ func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
 }
 
 func FileSize(entry *filer_pb.Entry) (size uint64) {
-	return maxUint64(TotalSize(entry.Chunks), entry.Attributes.FileSize)
+	if entry == nil || entry.Attributes == nil {
+		return 0
+	}
+	fileSize := entry.Attributes.FileSize
+	if entry.RemoteEntry != nil {
+		if entry.RemoteEntry.RemoteMtime > entry.Attributes.Mtime {
+			fileSize = maxUint64(fileSize, uint64(entry.RemoteEntry.RemoteSize))
+		}
+	}
+	return maxUint64(TotalSize(entry.Chunks), fileSize)
 }
 
 func ETag(entry *filer_pb.Entry) (etag string) {
@@ -245,19 +254,17 @@ func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunction
 	if true {
 		return visibles2, err
 	}
-
-	sort.Slice(chunks, func(i, j int) bool {
-		if chunks[i].Mtime == chunks[j].Mtime {
-			filer_pb.EnsureFid(chunks[i])
-			filer_pb.EnsureFid(chunks[j])
-			if chunks[i].Fid == nil || chunks[j].Fid == nil {
+	slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
+		if a.Mtime == b.Mtime {
+			filer_pb.EnsureFid(a)
+			filer_pb.EnsureFid(b)
+			if a.Fid == nil || b.Fid == nil {
 				return true
 			}
-			return chunks[i].Fid.FileKey < chunks[j].Fid.FileKey
+			return a.Fid.FileKey < b.Fid.FileKey
 		}
-		return chunks[i].Mtime < chunks[j].Mtime // keep this to make tests run
+		return a.Mtime < b.Mtime
 	})
-
 	for _, chunk := range chunks {
 
 		// glog.V(0).Infof("merge [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size))

weed/filer/filechunks2_test.go

@@ -1,7 +1,7 @@
 package filer
 
 import (
-	"sort"
+	"golang.org/x/exp/slices"
 	"testing"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -34,11 +34,11 @@ func TestCompactFileChunksRealCase(t *testing.T) {
 }
 
 func printChunks(name string, chunks []*filer_pb.FileChunk) {
-	sort.Slice(chunks, func(i, j int) bool {
-		if chunks[i].Offset == chunks[j].Offset {
-			return chunks[i].Mtime < chunks[j].Mtime
+	slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
+		if a.Offset == b.Offset {
+			return a.Mtime < b.Mtime
 		}
-		return chunks[i].Offset < chunks[j].Offset
+		return a.Offset < b.Offset
 	})
 	for _, chunk := range chunks {
 		glog.V(0).Infof("%s chunk %s [%10d,%10d)", name, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))

weed/filer/filechunks_read.go

@@ -2,7 +2,7 @@ package filer
 
 import (
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"sort"
+	"golang.org/x/exp/slices"
 )
 
 func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) {
@@ -22,17 +22,14 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk) (visibles []VisibleInterva
 			isStart: false,
 		})
 	}
-	sort.Slice(points, func(i, j int) bool {
-		if points[i].x != points[j].x {
-			return points[i].x < points[j].x
+	slices.SortFunc(points, func(a, b *Point) bool {
+		if a.x != b.x {
+			return a.x < b.x
 		}
-		if points[i].ts != points[j].ts {
-			return points[i].ts < points[j].ts
+		if a.ts != b.ts {
+			return a.ts < b.ts
 		}
-		if !points[i].isStart {
-			return true
-		}
-		return false
+		return !a.isStart
 	})
 
 	var prevX int64

weed/filer/filer.go

@@ -49,10 +49,10 @@ type Filer struct {
 	UniqueFileId        uint32
 }
 
-func NewFiler(masters []pb.ServerAddress, grpcDialOption grpc.DialOption,
-	filerHost pb.ServerAddress, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
+func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress,
+	filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
 	f := &Filer{
-		MasterClient:        wdclient.NewMasterClient(grpcDialOption, cluster.FilerType, filerHost, dataCenter, masters),
+		MasterClient:        wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, masters),
 		fileIdDeletionQueue: util.NewUnboundedQueue(),
 		GrpcDialOption:      grpcDialOption,
 		FilerConf:           NewFilerConf(),
@@ -84,6 +84,7 @@ func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNod
 	if grpcErr := pb.WithMasterClient(false, f.MasterClient.GetMaster(), f.GrpcDialOption, func(client master_pb.SeaweedClient) error {
 		resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
 			ClientType: cluster.FilerType,
+			FilerGroup: f.MasterClient.FilerGroup,
 		})
 
 		glog.V(0).Infof("the cluster has %d filers\n", len(resp.ClusterNodes))
@@ -151,7 +152,7 @@ func (f *Filer) RollbackTransaction(ctx context.Context) error {
 	return f.Store.RollbackTransaction(ctx)
 }
 
-func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32) error {
+func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool) error {
 
 	if string(entry.FullPath) == "/" {
 		return nil
@@ -169,9 +170,11 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFr
 
 	if oldEntry == nil {
 
-		dirParts := strings.Split(string(entry.FullPath), "/")
-		if err := f.ensureParentDirecotryEntry(ctx, entry, dirParts, len(dirParts)-1, isFromOtherCluster); err != nil {
-			return err
+		if !skipCreateParentDir {
+			dirParts := strings.Split(string(entry.FullPath), "/")
+			if err := f.ensureParentDirecotryEntry(ctx, entry, dirParts, len(dirParts)-1, isFromOtherCluster); err != nil {
+				return err
+			}
 		}
 
 		glog.V(4).Infof("InsertEntry %s: new entry: %v", entry.FullPath, entry.Name())

weed/filer/filer_delete_entry.go

@@ -25,9 +25,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 	if findErr != nil {
 		return findErr
 	}
-
 	isDeleteCollection := f.isBucket(entry)
-
 	if entry.IsDirectory() {
 		// delete the folder children, not including the folder itself
 		err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isDeleteCollection, isFromOtherCluster, signatures, func(chunks []*filer_pb.FileChunk) error {

weed/filer/filer_notify_append.go

@@ -43,7 +43,7 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error {
 	entry.Chunks = append(entry.Chunks, uploadResult.ToPbFileChunk(assignResult.Fid, offset))
 
 	// update the entry
-	err = f.CreateEntry(context.Background(), entry, false, false, nil)
+	err = f.CreateEntry(context.Background(), entry, false, false, nil, false)
 
 	return err
 }

weed/filer/filerstore.go

@@ -7,6 +7,8 @@ import (
 	"io"
 )
 
+const CountEntryChunksForGzip = 50
+
 var (
 	ErrUnsupportedListDirectoryPrefixed      = errors.New("unsupported directory prefix listing")
 	ErrUnsupportedSuperLargeDirectoryListing = errors.New("unsupported super large directory listing")

weed/filer/hbase/hbase_store.go

@@ -75,7 +75,7 @@ func (store *HbaseStore) InsertEntry(ctx context.Context, entry *filer.Entry) er
 	if err != nil {
 		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
 	}
-	if len(entry.Chunks) > 50 {
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
 		value = util.MaybeGzipData(value)
 	}
 

weed/filer/leveldb/leveldb_store.go

@@ -86,7 +86,7 @@ func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry)
 		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
 	}
 
-	if len(entry.Chunks) > 50 {
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
 		value = weed_util.MaybeGzipData(value)
 	}
 

weed/filer/leveldb/leveldb_store_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil)
 	dir := t.TempDir()
 	store := &LevelDBStore{}
 	store.initialize(dir)
@@ -31,7 +31,7 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil {
+	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}
@@ -65,7 +65,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil)
 	dir := t.TempDir()
 	store := &LevelDBStore{}
 	store.initialize(dir)
@@ -87,7 +87,7 @@ func TestEmptyRoot(t *testing.T) {
 }
 
 func BenchmarkInsertEntry(b *testing.B) {
-	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil)
 	dir := b.TempDir()
 	store := &LevelDBStore{}
 	store.initialize(dir)

weed/filer/leveldb2/leveldb2_store.go

@@ -88,7 +88,7 @@ func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer.Entry)
 		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
 	}
 
-	if len(entry.Chunks) > 50 {
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
 		value = weed_util.MaybeGzipData(value)
 	}
 

weed/filer/leveldb2/leveldb2_store_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil)
 	dir := t.TempDir()
 	store := &LevelDB2Store{}
 	store.initialize(dir, 2)
@@ -28,7 +28,7 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil {
+	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}
@@ -62,7 +62,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil)
 	dir := t.TempDir()
 	store := &LevelDB2Store{}
 	store.initialize(dir, 2)

weed/filer/leveldb3/leveldb3_store.go

@@ -177,7 +177,7 @@ func (store *LevelDB3Store) InsertEntry(ctx context.Context, entry *filer.Entry)
 		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
 	}
 
-	if len(entry.Chunks) > 50 {
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
 		value = weed_util.MaybeGzipData(value)
 	}
 

weed/filer/leveldb3/leveldb3_store_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil)
 	dir := t.TempDir()
 	store := &LevelDB3Store{}
 	store.initialize(dir)
@@ -28,7 +28,7 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil {
+	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}
@@ -62,7 +62,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", "", nil)
 	dir := t.TempDir()
 	store := &LevelDB3Store{}
 	store.initialize(dir)

weed/filer/meta_aggregator.go

@@ -76,9 +76,6 @@ func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) (no
 		}
 	} else {
 		if _, found := ma.peerStatues[address]; found {
-			ma.peerStatues[address] -= 1
-		}
-		if ma.peerStatues[address] <= 0 {
 			delete(ma.peerStatues, address)
 		}
 	}

weed/filer/mongodb/mongodb_store.go

@@ -107,7 +107,7 @@ func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry)
 		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
 	}
 
-	if len(entry.Chunks) > 50 {
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
 		meta = util.MaybeGzipData(meta)
 	}
 

weed/filer/read_write.go

@@ -62,6 +62,7 @@ func SaveInsideFiler(client filer_pb.SeaweedFilerClient, dir, name string, conte
 				},
 				Content: content,
 			},
+			SkipCheckParentDirectory: true,
 		})
 	} else if err == nil {
 		entry := resp.Entry

weed/filer/redis/universal_redis_store.go

@@ -3,7 +3,7 @@ package redis
 import (
 	"context"
 	"fmt"
-	"sort"
+	"golang.org/x/exp/slices"
 	"strings"
 	"time"
 
@@ -40,7 +40,7 @@ func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer.
 		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
 	}
 
-	if len(entry.Chunks) > 50 {
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
 		value = util.MaybeGzipData(value)
 	}
 
@@ -157,8 +157,8 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirP
 	}
 
 	// sort
-	sort.Slice(members, func(i, j int) bool {
-		return strings.Compare(members[i], members[j]) < 0
+	slices.SortFunc(members, func(a, b string) bool {
+		return strings.Compare(a, b) < 0
 	})
 
 	// limit

weed/filer/redis2/universal_redis_store.go

@@ -52,7 +52,7 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer
 		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
 	}
 
-	if len(entry.Chunks) > 50 {
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
 		value = util.MaybeGzipData(value)
 	}
 

weed/filer/redis3/universal_redis_store.go

@@ -40,7 +40,7 @@ func (store *UniversalRedis3Store) InsertEntry(ctx context.Context, entry *filer
 		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
 	}
 
-	if len(entry.Chunks) > 50 {
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
 		value = util.MaybeGzipData(value)
 	}
 

weed/filer/redis_lua/universal_redis_store.go

@@ -53,7 +53,7 @@ func (store *UniversalRedisLuaStore) InsertEntry(ctx context.Context, entry *fil
 		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
 	}
 
-	if len(entry.Chunks) > 50 {
+	if len(entry.Chunks) > filer.CountEntryChunksForGzip {
 		value = util.MaybeGzipData(value)
 	}
 

weed/filer/rocksdb/rocksdb_store_test.go

@@ -34,7 +34,7 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil); err != nil {
+	if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}

weed/filer/sqlite/doc.go

@@ -0,0 +1,9 @@
+/*
+
+Package sqlite is for sqlite filer store.
+
+The referenced "modernc.org/sqlite" library is too big when compiled.
+So this is only compiled in "make full_install".
+
+*/
+package sqlite

weed/filer/sqlite/sqlite_store.go

@@ -1,5 +1,6 @@
-//go:build linux || darwin || windows
+//go:build (linux || darwin || windows) && sqlite
 // +build linux darwin windows
+// +build sqlite
 
 // limited GOOS due to modernc.org/libc/unistd
 

weed/filer/sqlite/sqlite_store_unsupported.go

@@ -1,5 +1,5 @@
-//go:build !linux && !darwin && !windows && !s390 && !ppc64le && !mips64
-// +build !linux,!darwin,!windows,!s390,!ppc64le,!mips64
+//go:build !linux && !darwin && !windows && !s390 && !ppc64le && !mips64 && !sqlite
+// +build !linux,!darwin,!windows,!s390,!ppc64le,!mips64,!sqlite
 
 // limited GOOS due to modernc.org/libc/unistd
 

weed/filer/store_test/test_suite.go

@@ -0,0 +1,55 @@
+package store_test
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func TestFilerStore(t *testing.T, store filer.FilerStore) {
+	ctx := context.Background()
+
+	store.InsertEntry(ctx, makeEntry(util.FullPath("/"), true))
+	store.InsertEntry(ctx, makeEntry(util.FullPath("/a"), true))
+	store.InsertEntry(ctx, makeEntry(util.FullPath("/a/b"), true))
+	store.InsertEntry(ctx, makeEntry(util.FullPath("/a/b/c"), true))
+	for i := 0; i < 2000; i++ {
+		store.InsertEntry(ctx, makeEntry(util.FullPath(fmt.Sprintf("/a/b/c/f%05d", i)), false))
+	}
+
+	{
+		var counter int
+		lastFileName, err := store.ListDirectoryEntries(ctx, util.FullPath("/a/b/c"), "", false, 3, func(entry *filer.Entry) bool {
+			counter++
+			return true
+		})
+		assert.Nil(t, err, "list directory")
+		assert.Equal(t, 3, counter, "directory list counter")
+		assert.Equal(t, "f00003", lastFileName, "directory list last file")
+		lastFileName, err = store.ListDirectoryEntries(ctx, util.FullPath("/a/b/c"), lastFileName, false, 1024, func(entry *filer.Entry) bool {
+			counter++
+			return true
+		})
+		assert.Nil(t, err, "list directory")
+		assert.Equal(t, 1027, counter, "directory list counter")
+		assert.Equal(t, "f01027", lastFileName, "directory list last file")
+	}
+
+}
+
+func makeEntry(fullPath util.FullPath, isDirectory bool) *filer.Entry {
+	var mode os.FileMode
+	if isDirectory {
+		mode = os.ModeDir
+	}
+	return &filer.Entry{
+		FullPath: fullPath,
+		Attr: filer.Attr{
+			Mode: mode,
+		},
+	}
+}

weed/filer/stream.go

@@ -3,6 +3,7 @@ package filer
 import (
 	"bytes"
 	"fmt"
+	"golang.org/x/exp/slices"
 	"io"
 	"math"
 	"sort"
@@ -39,11 +40,11 @@ func isSameChunks(a, b []*filer_pb.FileChunk) bool {
 	if len(a) != len(b) {
 		return false
 	}
-	sort.Slice(a, func(i, j int) bool {
-		return strings.Compare(a[i].ETag, a[j].ETag) < 0
+	slices.SortFunc(a, func(i, j *filer_pb.FileChunk) bool {
+		return strings.Compare(i.ETag, j.ETag) < 0
 	})
-	sort.Slice(b, func(i, j int) bool {
-		return strings.Compare(b[i].ETag, b[j].ETag) < 0
+	slices.SortFunc(b, func(i, j *filer_pb.FileChunk) bool {
+		return strings.Compare(i.ETag, j.ETag) < 0
 	})
 	for i := 0; i < len(a); i++ {
 		if a[i].ETag != b[i].ETag {
@@ -62,7 +63,7 @@ func NewFileReader(filerClient filer_pb.FilerClient, entry *filer_pb.Entry) io.R
 
 func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
 
-	glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks)
+	glog.V(4).Infof("start to stream content for chunks: %+v", chunks)
 	chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
 
 	fileId2Url := make(map[string][]string)
@@ -104,10 +105,12 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writ
 		}
 		stats.FilerRequestCounter.WithLabelValues("chunkDownload").Inc()
 	}
-	glog.V(4).Infof("zero [%d,%d)", offset, offset+remaining)
-	err := writeZero(writer, remaining)
-	if err != nil {
-		return fmt.Errorf("write zero [%d,%d)", offset, offset+remaining)
+	if remaining > 0 {
+		glog.V(4).Infof("zero [%d,%d)", offset, offset+remaining)
+		err := writeZero(writer, remaining)
+		if err != nil {
+			return fmt.Errorf("write zero [%d,%d)", offset, offset+remaining)
+		}
 	}
 
 	return nil
@@ -177,8 +180,8 @@ var _ = io.ReaderAt(&ChunkStreamReader{})
 func doNewChunkStreamReader(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
 
 	chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
-	sort.Slice(chunkViews, func(i, j int) bool {
-		return chunkViews[i].LogicOffset < chunkViews[j].LogicOffset
+	slices.SortFunc(chunkViews, func(a, b *ChunkView) bool {
+		return a.LogicOffset < b.LogicOffset
 	})
 
 	var totalSize int64

weed/filer/ydb/doc.go

@@ -0,0 +1,9 @@
+/*
+
+Package ydb is for YDB filer store.
+
+The referenced "github.com/ydb-platform/ydb-go-sdk/v3" library is too big when compiled.
+So this is only compiled in "make full_install".
+
+*/
+package ydb