3.68

Bumps [docker/login-action](https://github.com/docker/login-action) from 3.1.0 to 3.2.0.
- [Release notes](https://github.com/docker/login-action/releases)
- [Commits](https://github.com/docker/login-action/compare/e92390c5fb421da1463c202d546fed0ec5c39f20...0d4c9c5ea7693da7b068278f7b52bda2a190a446)

---
updated-dependencies:
- dependency-name: docker/login-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/binaries_dev.yml

@@ -38,13 +38,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -60,7 +60,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -87,13 +87,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -109,7 +109,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release0.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release1.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release2.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release3.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release4.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -45,7 +45,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/codeql.yml

@@ -18,7 +18,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL

.github/workflows/container_dev.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -36,27 +36,27 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_latest.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -37,27 +37,27 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release1.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -37,17 +37,17 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release2.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -38,17 +38,17 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release3.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -38,17 +38,17 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release4.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -37,17 +37,17 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release5.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -37,17 +37,17 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/depsreview.yml

@@ -9,6 +9,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@fd07d42ce87ab09f10c61a2d1a5e59e6c655620a
+        uses: actions/dependency-review-action@0659a74c94536054bfa5aeb92241f70d680cc78e

.github/workflows/e2e.yml

@@ -30,7 +30,7 @@ jobs:
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
 
     - name: Install dependencies
       run: |

.github/workflows/go.yml

@@ -27,7 +27,7 @@ jobs:
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
 
     - name: Get dependencies
       run: |

.github/workflows/helm_chart_release.yml

@@ -12,7 +12,7 @@ jobs:
   release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: Publish Helm charts
         uses: stefanprodan/helm-gh-pages@master
         with:          

.github/workflows/helm_ci.yml

@@ -16,12 +16,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
         with:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3
+        uses: azure/setup-helm@v4
         with:
           version: v3.10.0
 
@@ -45,7 +45,7 @@ jobs:
         run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts
 
       - name: Create kind cluster
-        uses: helm/kind-action@v1.9.0
+        uses: helm/kind-action@v1.10.0
 
       - name: Run chart-testing (install)
         run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts

.github/workflows/s3tests.yml

@@ -0,0 +1,200 @@
+name: "Ceph S3 tests"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+concurrency:
+  group: ${{ github.head_ref }}/s3tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    working-directory: docker
+
+jobs:
+  s3tests:
+    name: Ceph S3 tests
+    runs-on: ubuntu-22.04
+    container:
+      image: docker.io/kmlebedev/ceph-s3-tests:0.0.2
+    timeout-minutes: 30
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Run Ceph S3 tests
+        timeout-minutes: 15
+        env:
+          S3TEST_CONF: /__w/seaweedfs/seaweedfs/docker/compose/s3tests.conf
+        shell: bash
+        run: |
+          cd /__w/seaweedfs/seaweedfs/weed
+          go install -buildvcs=false
+          set -x
+          nohup weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
+            -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
+            -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
+            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
+          sleep 10          
+          cd /s3-tests
+          tox -- \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_many \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_many \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_dot \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_one \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_one \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_zero \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_not_in_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_after_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_after_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_long_name \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \
+          s3tests_boto3/functional/test_s3.py::test_bucket_delete_notexist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_create_delete \
+          s3tests_boto3/functional/test_s3.py::test_object_read_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_multi_object_delete \
+          s3tests_boto3/functional/test_s3.py::test_multi_objectv2_delete \
+          s3tests_boto3/functional/test_s3.py::test_object_head_zero_bytes \
+          s3tests_boto3/functional/test_s3.py::test_object_write_check_etag \
+          s3tests_boto3/functional/test_s3.py::test_object_write_cache_control \
+          s3tests_boto3/functional/test_s3.py::test_object_write_expires \
+          s3tests_boto3/functional/test_s3.py::test_object_write_read_update_read_delete \
+          s3tests_boto3/functional/test_s3.py::test_object_metadata_replaced_on_put \
+          s3tests_boto3/functional/test_s3.py::test_object_write_file \
+          s3tests_boto3/functional/test_s3.py::test_post_object_invalid_date_format \
+          s3tests_boto3/functional/test_s3.py::test_post_object_no_key_specified \
+          s3tests_boto3/functional/test_s3.py::test_post_object_missing_signature \
+          s3tests_boto3/functional/test_s3.py::test_post_object_condition_is_case_sensitive \
+          s3tests_boto3/functional/test_s3.py::test_post_object_expires_is_case_sensitive \
+          s3tests_boto3/functional/test_s3.py::test_post_object_missing_expires_condition \
+          s3tests_boto3/functional/test_s3.py::test_post_object_missing_conditions_list \
+          s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_limit_exceeded \
+          s3tests_boto3/functional/test_s3.py::test_post_object_missing_content_length_argument \
+          s3tests_boto3/functional/test_s3.py::test_post_object_invalid_content_length_argument \
+          s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_below_minimum \
+          s3tests_boto3/functional/test_s3.py::test_post_object_empty_conditions \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_good \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_good \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_failed \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_failed \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
+          s3tests_boto3/functional/test_s3.py::test_bucket_head \
+          s3tests_boto3/functional/test_s3.py::test_bucket_head_notexist \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_bucket_acl \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_acl \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_gone \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero \
+          s3tests_boto3/functional/test_s3.py::test_object_anon_put \
+          s3tests_boto3/functional/test_s3.py::test_object_put_authenticated \
+          s3tests_boto3/functional/test_s3.py::test_bucket_recreate_overwrite_acl \
+          s3tests_boto3/functional/test_s3.py::test_bucket_recreate_new_acl \
+          s3tests_boto3/functional/test_s3.py::test_buckets_create_then_list \
+          s3tests_boto3/functional/test_s3.py::test_buckets_list_ctime \
+          s3tests_boto3/functional/test_s3.py::test_list_buckets_invalid_auth \
+          s3tests_boto3/functional/test_s3.py::test_list_buckets_bad_auth \
+          s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_period \
+          s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_hyphen \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_zero_size \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_same_bucket \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_to_itself \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_diff_bucket \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_canned_acl \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_bucket_not_found \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_key_not_found \
+          s3tests_boto3/functional/test_s3.py::test_multipart_copy_small \
+          s3tests_boto3/functional/test_s3.py::test_multipart_copy_without_range \
+          s3tests_boto3/functional/test_s3.py::test_multipart_copy_special_names \
+          s3tests_boto3/functional/test_s3.py::test_multipart_copy_multiple_sizes \
+          s3tests_boto3/functional/test_s3.py::test_multipart_get_part \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_empty \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_multiple_sizes \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_contents \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_size_too_small \
+          s3tests_boto3/functional/test_s3.py::test_multipart_resend_first_finishes_last \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_resend_part \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_missing_part \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_incorrect_etag \
+          s3tests_boto3/functional/test_s3.py::test_abort_multipart_upload \
+          s3tests_boto3/functional/test_s3.py::test_list_multipart_upload \
+          s3tests_boto3/functional/test_s3.py::test_atomic_read_1mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_read_4mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_read_8mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_write_1mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_write_4mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_write_8mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_1mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_4mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_8mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_multipart_upload_write \
+          s3tests_boto3/functional/test_s3.py::test_ranged_request_response_code \
+          s3tests_boto3/functional/test_s3.py::test_ranged_big_request_response_code \
+          s3tests_boto3/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \
+          s3tests_boto3/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \
+          s3tests_boto3/functional/test_s3.py::test_copy_object_ifmatch_good \
+          s3tests_boto3/functional/test_s3.py::test_copy_object_ifnonematch_failed \
+          s3tests_boto3/functional/test_s3.py::test_lifecycle_set \
+          s3tests_boto3/functional/test_s3.py::test_lifecycle_get \
+          s3tests_boto3/functional/test_s3.py::test_lifecycle_set_filter

.gitignore

@@ -87,8 +87,15 @@ other/java/hdfs/dependency-reduced-pom.xml
 
 # binary file
 weed/weed
-weed/mq/client/cmd/weed_pub/weed_pub
+weed/mq/client/cmd/weed_pub_kv/weed_pub
 docker/weed
 
 # test generated files
-weed/*/*.jpg
+weed/*/*.jpg
+docker/weed_sub
+docker/weed_pub
+weed/mq/schema/example.parquet
+docker/weed_pub_kv
+docker/weed_pub_record
+docker/weed_sub_kv
+docker/weed_sub_record

Makefile

@@ -15,7 +15,7 @@ full_install:
 	cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone"
 
 server: install
-	weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json -metricsPort=9324
+	weed -v 0 server -s3 -filer -filer.maxMB=64 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=./docker/compose/s3.json -metricsPort=9324
 
 benchmark: install warp_install
 	pkill weed || true

docker/Dockerfile.local

@@ -1,6 +1,8 @@
 FROM alpine AS final
 LABEL author="Chris Lu"
 COPY  ./weed /usr/bin/
+COPY  ./weed_pub* /usr/bin/
+COPY  ./weed_sub* /usr/bin/
 RUN mkdir -p /etc/seaweedfs
 COPY ./filer.toml /etc/seaweedfs/filer.toml
 COPY ./entrypoint.sh /entrypoint.sh

docker/Makefile

@@ -9,6 +9,10 @@ binary:
 	export SWCOMMIT=$(shell git rev-parse --short HEAD)
 	export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
 	cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w -extldflags -static $(SWLDFLAGS)" && mv weed ../docker/
+	cd ../weed/mq/client/cmd/weed_pub_kv && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_pub_kv ../../../../../docker/
+	cd ../weed/mq/client/cmd/weed_pub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_pub_record ../../../../../docker/
+	cd ../weed/mq/client/cmd/weed_sub_kv && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_sub_kv ../../../../../docker/
+	cd ../weed/mq/client/cmd/weed_sub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_sub_record ../../../../../docker/
 
 binary_race: options = -race
 binary_race: cgo = 1
@@ -81,12 +85,18 @@ cluster: build
 2mount: build
 	docker compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
 
+filer_backup: build
+	docker compose -f compose/local-filer-backup-compose.yml -p seaweedfs up
+
 hashicorp_raft: build
 	docker compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
 
 s3tests: build s3tests_build
 	docker compose -f compose/local-s3tests-compose.yml -p seaweedfs up
 
+brokers: build
+	docker compose -f compose/local-brokers-compose.yml -p seaweedfs up
+
 filer_etcd: build
 	docker stack deploy -c compose/swarm-etcd.yml fs
 

docker/compose/local-brokers-compose.yml

@@ -0,0 +1,127 @@
+version: '3.9'
+
+services:
+  master0:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "-v=0 master -volumeSizeLimitMB 100 -resumeState=false -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+  master1:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9334:9334
+      - 19334:19334
+    command: "-v=0 master -volumeSizeLimitMB 100 -resumeState=false -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+  master2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9335:9335
+      - 19335:19335
+    command: "-v=0 master -volumeSizeLimitMB 100 -resumeState=false -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+  volume1:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
+    depends_on:
+      - master0
+      - master1
+      - master2
+  volume2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8082:8082
+      - 18082:18082
+    command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
+    depends_on:
+      - master0
+      - master1
+      - master2
+  volume3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8083:8083
+      - 18083:18083
+    command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
+    depends_on:
+      - master0
+      - master1
+      - master2
+  filer1:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+    command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335" -port=8888 -ip=filer1'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume1
+      - volume2
+  filer2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8889:8889
+      - 18889:18889
+    command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335" -port=8889 -ip=filer2'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume1
+      - volume2
+      - filer1
+  broker1:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 17777:17777
+    command: 'mq.broker -master="master0:9333,master1:9334,master2:9335" -port=17777 -ip=broker1'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume1
+      - volume2
+      - filer1
+      - filer2
+  broker2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 17778:17778
+    command: 'mq.broker -master="master0:9333,master1:9334,master2:9335" -port=17778 -ip=broker2'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume1
+      - volume2
+      - filer1
+      - filer2
+  broker3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 17779:17779
+    command: 'mq.broker -master="master0:9333,master1:9334,master2:9335" -port=17779 -ip=broker3'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume1
+      - volume2
+      - filer1
+      - filer2

docker/compose/local-filer-backup-compose.yml

@@ -0,0 +1,54 @@
+version: '3.9'
+
+services:
+  server-left:
+    image: chrislusf/seaweedfs:local
+    command: "-v=0 server -ip=server-left -filer -filer.maxMB 5 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
+    healthcheck:
+      test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ]
+      interval: 3s
+      start_period: 15s
+      timeout: 30s
+  server-right:
+    image: chrislusf/seaweedfs:local
+    command: "-v=0 server -ip=server-right -filer -filer.maxMB 64 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
+    healthcheck:
+      test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ]
+      interval: 3s
+      start_period: 15s
+      timeout: 30s
+  filer-backup:
+    image: chrislusf/seaweedfs:local
+    command: "-v=0 filer.backup -debug -doDeleteFiles=False -filer server-left:8888"
+    volumes:
+      - ./replication.toml:/etc/seaweedfs/replication.toml
+    environment:
+      WEED_SINK_LOCAL_INCREMENTAL_ENABLED: "false"
+      WEED_SINK_S3_ENABLED: "true"
+      WEED_SINK_S3_BUCKET: "backup"
+      WEED_SINK_S3_ENDPOINT: "http://server-right:8333"
+      WEED_SINK_S3_DIRECTORY: "/"
+      WEED_SINK_S3_AWS_ACCESS_KEY_ID: "some_access_key1"
+      WEED_SINK_S3_AWS_SECRET_ACCESS_KEY: "some_secret_key1"
+      WEED_SINK_S3_S3_DISABLE_CONTENT_MD5_VALIDATION: "false"
+      WEED_SINK_S3_UPLOADER_PART_SIZE_MB: "5"
+      WEED_SINK_S3_KEEP_PART_SIZE: "false"
+    depends_on:
+      server-left:
+        condition: service_healthy
+      server-right:
+        condition: service_healthy
+  minio-warp:
+    image: minio/warp
+    command: 'mixed --duration 5s --obj.size=6mb --md5 --objects 10 --concurrent 2'
+    restart: on-failure
+    environment:
+      WARP_HOST: "server-left:8333"
+      WARP_ACCESS_KEY: "some_access_key1"
+      WARP_SECRET_KEY: "some_secret_key1"
+    depends_on:
+      - filer-backup

docker/compose/s3tests.conf

@@ -2,7 +2,7 @@
 ## this section is just used for host, port and bucket_prefix
 
 # host set for rgw in vstart.sh
-host = s3
+host = 127.0.0.1
 
 # port set for rgw in vstart.sh
 port = 8000

docker/compose/test-etcd-filer.yml

@@ -30,6 +30,7 @@ services:
     environment:
       WEED_LEVELDB2_ENABLED: 'false'
       WEED_ETCD_ENABLED: 'true'
+      WEED_ETCD_KEY_PREFIX: 'seaweedfs.'
       WEED_ETCD_SERVERS: "http://etcd:2379"
     volumes:
       - ./s3.json:/etc/seaweedfs/s3.json

go.mod

@@ -3,22 +3,21 @@ module github.com/seaweedfs/seaweedfs
 go 1.22.0
 
 require (
-	cloud.google.com/go v0.112.0 // indirect
-	cloud.google.com/go/pubsub v1.36.1
-	cloud.google.com/go/storage v1.36.0
+	cloud.google.com/go v0.113.0 // indirect
+	cloud.google.com/go/pubsub v1.38.0
+	cloud.google.com/go/storage v1.41.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Shopify/sarama v1.38.1
-	github.com/aws/aws-sdk-go v1.50.10
+	github.com/aws/aws-sdk-go v1.53.5
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
-	github.com/cenkalti/backoff/v4 v4.2.1
+	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/disintegration/imaging v1.6.2
 	github.com/dustin/go-humanize v1.0.1
 	github.com/eapache/go-resiliency v1.3.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
@@ -28,22 +27,20 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
-	github.com/fclairamb/ftpserverlib v0.22.0
+	github.com/fclairamb/ftpserverlib v0.24.0
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/go-redsync/redsync/v4 v4.11.0
-	github.com/go-sql-driver/mysql v1.7.1
+	github.com/go-redsync/redsync/v4 v4.13.0
+	github.com/go-sql-driver/mysql v1.8.1
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.6.0
-	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3
+	github.com/golang/protobuf v1.5.4
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.2
 	github.com/google/uuid v1.6.0
-	github.com/google/wire v0.5.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
+	github.com/google/wire v0.6.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.4 // indirect
 	github.com/gorilla/mux v1.8.1
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -56,12 +53,11 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
-	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.17.3 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/klauspost/reedsolomon v1.12.1
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.9
-	github.com/linxGnu/grocksdb v1.8.12
+	github.com/linxGnu/grocksdb v1.8.14
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-ieproxy v0.0.11 // indirect
@@ -75,10 +71,10 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/posener/complete v1.2.3
 	github.com/pquerna/cachecontrol v0.2.0
-	github.com/prometheus/client_golang v1.18.0
+	github.com/prometheus/client_golang v1.19.0
 	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.45.0 // indirect
-	github.com/prometheus/procfs v0.12.0
+	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/procfs v0.15.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
@@ -87,94 +83,94 @@ require (
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/viper v1.18.2
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
-	github.com/tidwall/gjson v1.17.0
+	github.com/tidwall/gjson v1.17.1
 	github.com/tidwall/match v1.1.1
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
 	github.com/valyala/bytebufferpool v1.0.0
-	github.com/viant/assertly v0.5.4 // indirect
-	github.com/viant/ptrie v0.3.0
-	github.com/viant/toolbox v0.33.2 // indirect
+	github.com/viant/ptrie v1.0.1
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
-	go.etcd.io/etcd/client/v3 v3.5.11
-	go.mongodb.org/mongo-driver v1.13.1
+	go.etcd.io/etcd/client/v3 v3.5.13
+	go.mongodb.org/mongo-driver v1.15.0
 	go.opencensus.io v0.24.0 // indirect
-	gocloud.dev v0.36.0
-	gocloud.dev/pubsub/natspubsub v0.36.0
-	gocloud.dev/pubsub/rabbitpubsub v0.36.0
-	golang.org/x/crypto v0.19.0 // indirect
-	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
-	golang.org/x/image v0.15.0
-	golang.org/x/net v0.21.0
-	golang.org/x/oauth2 v0.17.0 // indirect
-	golang.org/x/sys v0.17.0
-	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/tools v0.16.1
+	gocloud.dev v0.37.0
+	gocloud.dev/pubsub/natspubsub v0.37.0
+	gocloud.dev/pubsub/rabbitpubsub v0.37.0
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3
+	golang.org/x/image v0.16.0
+	golang.org/x/net v0.25.0
+	golang.org/x/oauth2 v0.20.0 // indirect
+	golang.org/x/sys v0.20.0
+	golang.org/x/text v0.15.0 // indirect
+	golang.org/x/tools v0.21.0
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
-	google.golang.org/api v0.165.0
-	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe // indirect
-	google.golang.org/grpc v1.61.0
-	google.golang.org/protobuf v1.32.0
+	google.golang.org/api v0.181.0
+	google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect
+	google.golang.org/grpc v1.63.2
+	google.golang.org/protobuf v1.34.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
-	modernc.org/cc/v3 v3.40.0 // indirect
-	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.29.0 // indirect
+	modernc.org/libc v1.50.9 // indirect
 	modernc.org/mathutil v1.6.0
-	modernc.org/memory v1.7.2 // indirect
-	modernc.org/opt v0.1.3 // indirect
-	modernc.org/sqlite v1.28.0
+	modernc.org/memory v1.8.0 // indirect
+	modernc.org/sqlite v1.30.0
 	modernc.org/strutil v1.2.0
-	modernc.org/token v1.0.1 // indirect
+	modernc.org/token v1.1.0 // indirect
 )
 
 require (
 	github.com/Jille/raft-grpc-transport v1.5.0
-	github.com/arangodb/go-driver v1.6.1
+	github.com/arangodb/go-driver v1.6.2
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.24.1
-	github.com/aws/aws-sdk-go-v2/config v1.26.6
-	github.com/aws/aws-sdk-go-v2/credentials v1.16.16
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0
+	github.com/aws/aws-sdk-go-v2 v1.27.0
+	github.com/aws/aws-sdk-go-v2/config v1.27.16
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.16
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.54.3
+	github.com/cognusion/imaging v1.0.1
 	github.com/fluent/fluent-logger-golang v1.9.0
-	github.com/golang-jwt/jwt/v5 v5.1.0
+	github.com/getsentry/sentry-go v0.27.0
+	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
-	github.com/hanwen/go-fuse/v2 v2.5.0
-	github.com/hashicorp/raft v1.6.0
+	github.com/hanwen/go-fuse/v2 v2.5.1
+	github.com/hashicorp/raft v1.6.1
 	github.com/hashicorp/raft-boltdb/v2 v2.3.0
 	github.com/orcaman/concurrent-map/v2 v2.0.1
-	github.com/puzpuzpuz/xsync/v2 v2.5.1
+	github.com/parquet-go/parquet-go v0.22.0
 	github.com/rabbitmq/amqp091-go v1.9.0
-	github.com/rclone/rclone v1.65.2
-	github.com/rdleal/intervalst v1.2.2
-	github.com/schollz/progressbar/v3 v3.14.1
-	github.com/shirou/gopsutil/v3 v3.24.1
+	github.com/rclone/rclone v1.66.0
+	github.com/rdleal/intervalst v1.4.0
+	github.com/schollz/progressbar/v3 v3.14.3
+	github.com/shirou/gopsutil/v3 v3.24.4
 	github.com/tikv/client-go/v2 v2.0.7
-	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.56.1
+	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.4.2
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.68.1
+	go.etcd.io/etcd/client/pkg/v3 v3.5.14
 	go.uber.org/atomic v1.11.0
+	golang.org/x/sync v0.7.0
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
 )
 
 require (
-	cloud.google.com/go/compute v1.23.3 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.1.5 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.0 // indirect
+	cloud.google.com/go/auth v0.4.1 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
+	cloud.google.com/go/compute/metadata v0.3.0 // indirect
+	cloud.google.com/go/iam v1.1.8 // indirect
+	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.1 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
@@ -186,26 +182,27 @@ require (
 	github.com/PuerkitoBio/goquery v1.8.1 // indirect
 	github.com/Unknwon/goconfig v1.0.0 // indirect
 	github.com/abbot/go-http-auth v0.4.0 // indirect
+	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.26.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.29.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
-	github.com/aws/smithy-go v1.19.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.10 // indirect
+	github.com/aws/smithy-go v1.20.2 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
-	github.com/bradenaw/juniper v0.13.1 // indirect
+	github.com/bradenaw/juniper v0.15.2 // indirect
 	github.com/buengese/sgzip v0.1.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
@@ -217,16 +214,16 @@ require (
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
-	github.com/emersion/go-message v0.17.0 // indirect
+	github.com/emersion/go-message v0.18.0 // indirect
 	github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
 	github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect
 	github.com/fatih/color v1.14.1 // indirect
-	github.com/fclairamb/go-log v0.4.1 // indirect
+	github.com/fclairamb/go-log v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/flynn/noise v1.0.0 // indirect
+	github.com/flynn/noise v1.0.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
-	github.com/go-chi/chi/v5 v5.0.10 // indirect
+	github.com/go-chi/chi/v5 v5.0.11 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
@@ -234,17 +231,16 @@ require (
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
-	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
-	github.com/hashicorp/go-hclog v1.5.0 // indirect
+	github.com/hashicorp/go-hclog v1.6.2 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/henrybear327/Proton-API-Bridge v1.0.0 // indirect
 	github.com/henrybear327/go-proton-api v1.0.0 // indirect
-	github.com/iguanesolutions/go-systemd/v5 v5.1.1 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
@@ -253,9 +249,9 @@ require (
 	github.com/jonboulle/clockwork v0.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jtolio/eventkit v0.0.0-20231019094657-5d77ebb407d9 // indirect
-	github.com/jtolio/noiseconn v0.0.0-20230621152802-afeab29449e0 // indirect
+	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
@@ -263,39 +259,38 @@ require (
 	github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
-	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
-	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/montanaflynn/stats v0.7.0 // indirect
-	github.com/nats-io/nats.go v1.31.0 // indirect
-	github.com/nats-io/nkeys v0.4.6 // indirect
+	github.com/nats-io/nats.go v1.33.1 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/ncw/swift/v2 v2.0.2 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/onsi/gomega v1.27.10 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/oracle/oci-go-sdk/v65 v65.51.0 // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.55.1 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
 	github.com/philhofer/fwd v1.1.2 // indirect
-	github.com/pierrec/lz4/v4 v4.1.17 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
 	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
 	github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
-	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/sftp v1.13.6 // indirect
 	github.com/pkg/xattr v0.4.9 // indirect
 	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
 	github.com/relvacode/iso8601 v1.3.0 // indirect
 	github.com/rfjakob/eme v1.1.2 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/segmentio/encoding v0.3.6 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/smartystreets/goconvey v1.8.1 // indirect
@@ -304,49 +299,47 @@ require (
 	github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca // indirect
+	github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 // indirect
 	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
 	github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
 	github.com/tinylib/msgp v1.1.8 // indirect
-	github.com/tklauser/go-sysconf v0.3.12 // indirect
-	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/tklauser/go-sysconf v0.3.13 // indirect
+	github.com/tklauser/numcpus v0.7.0 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
-	github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
-	github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf // indirect
-	github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect
-	github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca // indirect
+	github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
+	github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
-	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
 	github.com/zeebo/errs v1.3.0 // indirect
 	go.etcd.io/bbolt v1.3.8 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.11 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.11 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
-	go.opentelemetry.io/otel v1.23.0 // indirect
-	go.opentelemetry.io/otel/metric v1.23.0 // indirect
-	go.opentelemetry.io/otel/trace v1.23.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.13 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.26.0 // indirect
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/term v0.17.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
+	golang.org/x/term v0.20.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	lukechampine.com/uint128 v1.2.0 // indirect
-	storj.io/common v0.0.0-20231027080355-b4cb1b0d728e // indirect
+	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
+	storj.io/common v0.0.0-20240111121419-ecae1362576c // indirect
 	storj.io/drpc v0.0.33 // indirect
+	storj.io/infectious v0.0.2 // indirect
 	storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c // indirect
-	storj.io/uplink v1.12.1 // indirect
+	storj.io/uplink v1.12.2 // indirect
 )
 
 // replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft

k8s/charts/seaweedfs/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.63"
-version: 3.63.0
+appVersion: "3.68"
+version: 3.68.0

k8s/charts/seaweedfs/README.md

@@ -30,11 +30,11 @@ so your deployment will be spread/HA.
 ## Prerequisites
 ### Database
 
-leveldb is the default database this only supports one filer replica.
+leveldb is the default database, this supports multiple filer replicas that will [sync automatically](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication), with some [limitations](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication#limitation).
 
-To have multiple filers a external datastore is recommened.
+When the [limitations](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication#limitation) apply, or for a large number of filer replicas, an external datastore is recommended.
 
-Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`. 
+Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
 This database should be pre-configured and initialized by running:
 ```sql
 CREATE TABLE IF NOT EXISTS `filemeta` (
@@ -96,7 +96,7 @@ filer:
     enabled: true
 ```
 
-### Enabling Authenticaion to S3
+### Enabling Authentication to S3
 
 To enable authentication for S3, you have two options:
 

k8s/charts/seaweedfs/templates/filer-ingress.yaml

@@ -11,9 +11,9 @@ kind: Ingress
 metadata:
   name: ingress-{{ template "seaweedfs.name" . }}-filer
   namespace: {{ .Release.Namespace }}
-  {{- if .Values.filer.ingress.annotations }}
+  {{- with .Values.filer.ingress.annotations }}
   annotations:
-    {{ tpl .Values.filer.ingress.annotations . | nindent 4 | trim }}
+    {{- toYaml . | nindent 4 }}
   {{- end }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}

k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml

@@ -12,6 +12,9 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/component: filer
+    {{- with .Values.global.monitoring.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
 spec:
   endpoints:
     - interval: 30s

k8s/charts/seaweedfs/templates/filer-statefulset.yaml

@@ -35,10 +35,16 @@ spec:
       {{ with .Values.podLabels }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.filer.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
       annotations:
       {{ with .Values.podAnnotations }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.filer.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.filer.restartPolicy }}
       {{- if .Values.filer.affinity }}
@@ -62,6 +68,9 @@ spec:
       initContainers:
         {{ tpl .Values.filer.initContainers . | nindent 8 | trim }}
       {{- end }}
+      {{- if .Values.filer.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.filer.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
       containers:
         - name: seaweedfs
           image: {{ template "filer.image" . }}
@@ -84,11 +93,13 @@ spec:
                 secretKeyRef:
                   name: secret-seaweedfs-db
                   key: user
+                  optional: true
             - name: WEED_MYSQL_PASSWORD
               valueFrom:
                 secretKeyRef:
                   name: secret-seaweedfs-db
                   key: password
+                  optional: true
             - name: SEAWEEDFS_FULLNAME
               value: "{{ template "seaweedfs.name" . }}"
             {{- if .Values.filer.extraEnvironmentVars }}
@@ -124,7 +135,7 @@ spec:
             - "-ec"
             - |
               exec /usr/bin/weed \
-              {{- if eq .Values.filer.logs.type "hostPath" }}
+              {{- if or (eq .Values.filer.logs.type "hostPath") (eq .Values.filer.logs.type "persistentVolumeClaim") (eq .Values.filer.logs.type "emptyDir") }}
               -logdir=/logs \
               {{- else }}
               -logtostderr=true \
@@ -192,7 +203,7 @@ spec:
               {{- end }}
               -master={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }}
           volumeMounts:
-            {{- if eq .Values.filer.logs.type "hostPath" }}
+            {{- if (or (eq .Values.filer.logs.type "hostPath") (eq .Values.filer.logs.type "persistentVolumeClaim") (eq .Values.filer.logs.type "emptyDir")) }}
             - name: seaweedfs-filer-log-volume
               mountPath: "/logs/"
             {{- end }}
@@ -201,7 +212,7 @@ spec:
               mountPath: /etc/sw
               readOnly: true
             {{- end }}
-            {{- if (or .Values.filer.enablePVC (or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.data.type "persistentVolumeClaim"))) }}
+            {{- if (or .Values.filer.enablePVC (or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.data.type "persistentVolumeClaim") (eq .Values.filer.data.type "emptyDir"))) }}
             - name: data-filer
               mountPath: /data
             {{- end }}
@@ -262,6 +273,9 @@ spec:
           resources:
             {{ tpl .Values.filer.resources . | nindent 12 | trim }}
           {{- end }}
+          {{- if .Values.filer.containerSecurityContext.enabled }}
+          securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
         {{- if .Values.filer.sidecars }}
         {{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
         {{- end }}
@@ -277,6 +291,10 @@ spec:
           persistentVolumeClaim:
             claimName: {{ .Values.filer.logs.claimName }}
         {{- end }}
+        {{- if eq .Values.filer.logs.type "emptyDir" }}
+        - name: seaweedfs-filer-log-volume
+          emptyDir: {}
+        {{- end }}
         {{- if eq .Values.filer.data.type "hostPath" }}
         - name: data-filer
           hostPath:
@@ -288,6 +306,10 @@ spec:
           persistentVolumeClaim:
             claimName: {{ .Values.filer.data.claimName }}
         {{- end }}
+        {{- if eq .Values.filer.data.type "emptyDir" }}
+        - name: data-filer
+          emptyDir: {}
+        {{- end }}
         - name: db-schema-config-volume
           configMap:
             name: seaweedfs-db-init-config

k8s/charts/seaweedfs/templates/master-ingress.yaml

@@ -11,9 +11,9 @@ kind: Ingress
 metadata:
   name: ingress-{{ template "seaweedfs.name" . }}-master
   namespace: {{ .Release.Namespace }}
-  {{- if .Values.master.ingress.annotations }}
+  {{- with .Values.master.ingress.annotations }}
   annotations:
-    {{ tpl .Values.master.ingress.annotations . | nindent 4 | trim }}
+    {{- toYaml . | nindent 4 }}
   {{- end }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}

k8s/charts/seaweedfs/templates/master-servicemonitor.yaml

@@ -12,6 +12,9 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/component: master
+    {{- with .Values.global.monitoring.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
 spec:
   endpoints:
     - interval: 30s

k8s/charts/seaweedfs/templates/master-statefulset.yaml

@@ -34,10 +34,16 @@ spec:
       {{ with .Values.podLabels }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.master.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
       annotations:
       {{ with .Values.podAnnotations }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.master.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.master.restartPolicy }}
       {{- if .Values.master.affinity }}
@@ -61,6 +67,9 @@ spec:
       initContainers:
         {{ tpl .Values.master.initContainers . | nindent 8 | trim }}
       {{- end }}
+      {{- if .Values.master.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.master.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
       containers:
         - name: seaweedfs
           image: {{ template "master.image" . }}
@@ -105,9 +114,9 @@ spec:
           command:
             - "/bin/sh"
             - "-ec"
-            - | 
+            - |
               exec /usr/bin/weed \
-              {{- if eq .Values.master.logs.type "hostPath" }}
+              {{- if or (eq .Values.master.logs.type "hostPath") (eq .Values.master.logs.type "persistentVolumeClaim") (eq .Values.master.logs.type "emptyDir") }}
               -logdir=/logs \
               {{- else }}
               -logtostderr=true \
@@ -155,7 +164,7 @@ spec:
           volumeMounts:
             - name : data-{{ .Release.Namespace }}
               mountPath: /data
-            {{- if eq .Values.master.logs.type "hostPath" }}
+            {{- if or (eq .Values.master.logs.type "hostPath") (eq .Values.master.logs.type "persistentVolumeClaim") (eq .Values.master.logs.type "emptyDir") }}
             - name: seaweedfs-master-log-volume
               mountPath: "/logs/"
             {{- end }}
@@ -222,6 +231,9 @@ spec:
           resources:
             {{ tpl .Values.master.resources . | nindent 12 | trim }}
           {{- end }}
+          {{- if .Values.master.containerSecurityContext.enabled }}
+          securityContext: {{- omit .Values.master.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
         {{- if .Values.master.sidecars }}
         {{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
         {{- end }}
@@ -235,7 +247,11 @@ spec:
         {{- if eq .Values.master.logs.type "existingClaim" }}
         - name: seaweedfs-master-log-volume
           persistentVolumeClaim:
-            claimName: {{ .Values.master.logs.claimName }} 
+            claimName: {{ .Values.master.logs.claimName }}
+        {{- end }}
+        {{- if eq .Values.master.logs.type "emptyDir" }}
+        - name: seaweedfs-master-log-volume
+          emptyDir: {}
         {{- end }}
         {{- if eq .Values.master.data.type "hostPath" }}
         - name: data-{{ .Release.Namespace }}
@@ -246,7 +262,11 @@ spec:
         {{- if eq .Values.master.data.type "existingClaim" }}
         - name: data-{{ .Release.Namespace }}
           persistentVolumeClaim:
-            claimName: {{ .Values.master.data.claimName }} 
+            claimName: {{ .Values.master.data.claimName }}
+        {{- end }}
+        {{- if eq .Values.master.data.type "emptyDir" }}
+        - name: data-{{ .Release.Namespace }}
+          emptyDir: {}
         {{- end }}
         - name: master-config
           configMap:

k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml

@@ -22,6 +22,9 @@ spec:
         app.kubernetes.io/instance: {{ .Release.Name | quote }}
     spec:
       restartPolicy: Never
+      {{- if .Values.filer.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.filer.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
       containers:
       - name: post-install-job
         image: {{ template "master.image" . }}
@@ -80,6 +83,9 @@ spec:
           {{- end }}
           - containerPort: {{ .Values.master.grpcPort }}
             #name: swfs-master-grpc
+        {{- if .Values.filer.containerSecurityContext.enabled }}
+        securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+        {{- end }}
     {{- if .Values.filer.s3.enableAuth }}
       volumes:
         - name: config-users

k8s/charts/seaweedfs/templates/s3-deployment.yaml

@@ -27,10 +27,16 @@ spec:
       {{ with .Values.podLabels }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.s3.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
       annotations:
       {{ with .Values.podAnnotations }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.s3.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.s3.restartPolicy }}
       {{- if .Values.s3.tolerations }}
@@ -50,6 +56,9 @@ spec:
       initContainers:
         {{ tpl .Values.s3.initContainers . | nindent 8 | trim }}
       {{- end }}
+      {{- if .Values.s3.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.s3.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
       containers:
         - name: seaweedfs
           image: {{ template "s3.image" . }}
@@ -96,7 +105,7 @@ spec:
             - "-ec"
             - |
               exec /usr/bin/weed \
-               {{- if eq .Values.s3.logs.type "hostPath" }}
+              {{- if or (eq .Values.s3.logs.type "hostPath") (eq .Values.s3.logs.type "emptyDir") }}
               -logdir=/logs \
               {{- else }}
               -logtostderr=true \
@@ -133,7 +142,7 @@ spec:
               {{- end }}
               -filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }}
           volumeMounts:
-            {{- if eq .Values.s3.logs.type "hostPath" }}
+            {{- if or (eq .Values.s3.logs.type "hostPath") (eq .Values.s3.logs.type "emptyDir") }}
             - name: logs
               mountPath: "/logs/"
             {{- end }}
@@ -199,6 +208,9 @@ spec:
           resources:
             {{ tpl .Values.s3.resources . | nindent 12 | trim }}
           {{- end }}
+          {{- if .Values.s3.containerSecurityContext.enabled }}
+          securityContext: {{- omit .Values.s3.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
       {{- if .Values.s3.sidecars }}
       {{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
       {{- end }}
@@ -219,6 +231,10 @@ spec:
             path: {{ .Values.s3.logs.hostPathPrefix }}/logs/seaweedfs/s3
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.s3.logs.type "emptyDir" }}
+        - name: logs
+          emptyDir: {}
+        {{- end }}
         {{- if .Values.global.enableSecurity }}
         - name: security-config
           configMap:

k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml

@@ -12,6 +12,9 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/component: s3
+    {{- with .Values.global.monitoring.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
 spec:
   endpoints:
     - interval: 30s

k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml

@@ -12,6 +12,9 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/component: volume
+    {{- with .Values.global.monitoring.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
 spec:
   endpoints:
     - interval: 30s

k8s/charts/seaweedfs/templates/volume-statefulset.yaml

@@ -28,10 +28,16 @@ spec:
       {{ with .Values.podLabels }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.volume.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
       annotations:
       {{ with .Values.podAnnotations }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.volume.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       {{- if .Values.volume.affinity }}
       affinity:
@@ -72,6 +78,9 @@ spec:
         {{ tpl .Values.volume.initContainers . | nindent 8 | trim }}
         {{- end }}
       {{- end }}
+      {{- if .Values.volume.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.volume.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
       containers:
         - name: seaweedfs
           image: {{ template "volume.image" . }}
@@ -237,6 +246,9 @@ spec:
           resources:
             {{ tpl .Values.volume.resources . | nindent 12 | trim }}
           {{- end }}
+          {{- if .Values.volume.containerSecurityContext.enabled }}
+          securityContext: {{- omit .Values.volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
       {{- if .Values.volume.sidecars }}
       {{- include "common.tplvalues.render" (dict "value" .Values.volume.sidecars "context" $) | nindent 8 }}
       {{- end }}
@@ -255,6 +267,10 @@ spec:
           persistentVolumeClaim:
             claimName: {{ $dir.claimName }}
       {{- end }}
+      {{- if eq $dir.type "emptyDir" }}
+        - name: {{ $dir.name }}
+          emptyDir: {}
+      {{- end }}
 
      {{- end }}
 
@@ -270,6 +286,10 @@ spec:
           persistentVolumeClaim:
             claimName: {{ .Values.volume.idx.claimName }}
        {{- end }}
+       {{- if eq .Values.volume.idx.type "emptyDir" }}
+        - name: idx
+          emptyDir: {}
+       {{- end }}
      {{- end }}
 
      {{- if .Values.volume.logs }}
@@ -284,6 +304,10 @@ spec:
           persistentVolumeClaim:
             claimName: {{ .Values.volume.logs.claimName }}
        {{- end }}
+       {{- if eq .Values.volume.logs.type "emptyDir" }}
+        - name: logs
+          emptyDir: {}
+       {{- end }}
      {{- end }}
      {{- if .Values.global.enableSecurity }}
         - name: security-config

k8s/charts/seaweedfs/values.yaml

@@ -25,6 +25,7 @@ global:
     enabled: false
     gatewayHost: null
     gatewayPort: null
+    additionalLabels: {}
   # if enabled will use global.replicationPlacment and override master & filer defaultReplicaPlacement config
   enableReplication: false
   #  replication type is XYZ:
@@ -77,7 +78,7 @@ master:
 
   config: |-
     # Enter any extra configuration for master.toml here.
-    # It may be be a multi-line string.
+    # It may be a multi-line string.
 
   # You may use ANY storage-class, example with local-path-provisioner
   # Annotations are optional.
@@ -92,11 +93,18 @@ master:
   #  data:
   #    type: "existingClaim"
   #    claimName: "my-pvc"
+  #
+  # You can also use emptyDir storage:
+  #  data:
+  #    type: "emptyDir"
   data:
     type: "hostPath"
     storageClass: ""
     hostPathPrefix: /ssd
 
+  # You can also use emptyDir storage:
+  #  logs:
+  #    type: "emptyDir"
   logs:
     type: "hostPath"
     size: ""
@@ -119,6 +127,12 @@ master:
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  # Labels to be added to the master pods
+  podLabels: {}
+
+  # Annotations to be added to the master pods
+  podAnnotations: {}
+
   ## Set podManagementPolicy
   podManagementPolicy: Parallel
 
@@ -166,12 +180,31 @@ master:
   # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
   serviceAccountName: ""
 
+  # Configure security context for Pod
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # podSecurityContext:
+  #   enabled: true
+  #   runAsUser: 1000
+  #   runAsGroup: 3000
+  #   fsGroup: 2000
+  podSecurityContext: {}
+
+  # Configure security context for Container
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # containerSecurityContext:
+  #   enabled: true
+  #   runAsUser: 2000
+  #   allowPrivilegeEscalation: false
+  containerSecurityContext: {}
+
   ingress:
     enabled: false
     className: "nginx"
     # host: false for "*" hostname
     host: "master.seaweedfs.local"
-    annotations: |
+    annotations:
       nginx.ingress.kubernetes.io/auth-type: "basic"
       nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
       nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master'
@@ -255,9 +288,14 @@ volume:
   #     maxVolumes: 0  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
   #
   # You may also spacify an existing claim:
-  #    - name: data
-  #      type: "existingClaim"
-  #      claimName: "my-pvc"
+  #   - name: data
+  #     type: "existingClaim"
+  #     claimName: "my-pvc"
+  #     maxVolumes: 0  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
+  #
+  # You can also use emptyDir storage:
+  #   - name: data
+  #     type: "emptyDir"
   #     maxVolumes: 0  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
 
   dataDirs:
@@ -290,12 +328,17 @@ volume:
   # idx:
   #   type: "existingClaim"
   #   claimName: "myClaim"
+  #
+  # or
+  #
+  # idx:
+  #  type: "emptyDir"
 
   # same applies to "logs"
 
-  idx: ""
+  idx: {}
 
-  logs: ""
+  logs: {}
 
   # limit background compaction or copying speed in mega bytes per second
   compactionMBps: "50"
@@ -332,6 +375,12 @@ volume:
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  # Labels to be added to the volume pods
+  podLabels: {}
+
+  # Annotations to be added to the volume pods
+  podAnnotations: {}
+
   ## Set podManagementPolicy
   podManagementPolicy: Parallel
 
@@ -377,6 +426,25 @@ volume:
 
   extraEnvironmentVars:
 
+  # Configure security context for Pod
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # podSecurityContext:
+  #   enabled: true
+  #   runAsUser: 1000
+  #   runAsGroup: 3000
+  #   fsGroup: 2000
+  podSecurityContext: {}
+
+  # Configure security context for Container
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # containerSecurityContext:
+  #   enabled: true
+  #   runAsUser: 2000
+  #   allowPrivilegeEscalation: false
+  containerSecurityContext: {}
+
   # used to configure livenessProbe on volume-server containers
   #
   livenessProbe:
@@ -460,12 +528,19 @@ filer:
   #  data:
   #    type: "existingClaim"
   #    claimName: "my-pvc"
+  #
+  # You can also use emptyDir storage:
+  #  data:
+  #    type: "emptyDir"
   data:
     type: "hostPath"
     size: ""
     storageClass: ""
     hostPathPrefix: /storage
 
+  # You can also use emptyDir storage:
+  #  logs:
+  #    type: "emptyDir"
   logs:
     type: "hostPath"
     size: ""
@@ -488,6 +563,12 @@ filer:
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  # Labels to be added to the filer pods
+  podLabels: {}
+
+  # Annotations to be added to the filer pods
+  podAnnotations: {}
+
   ## Set podManagementPolicy
   podManagementPolicy: Parallel
 
@@ -535,12 +616,31 @@ filer:
   # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
   serviceAccountName: ""
 
+  # Configure security context for Pod
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # podSecurityContext:
+  #   enabled: true
+  #   runAsUser: 1000
+  #   runAsGroup: 3000
+  #   fsGroup: 2000
+  podSecurityContext: {}
+
+  # Configure security context for Container
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # containerSecurityContext:
+  #   enabled: true
+  #   runAsUser: 2000
+  #   allowPrivilegeEscalation: false
+  containerSecurityContext: {}
+
   ingress:
     enabled: false
     className: "nginx"
     # host: false for "*" hostname
     host: "seaweedfs.cluster.local"
-    annotations: |
+    annotations:
       nginx.ingress.kubernetes.io/backend-protocol: GRPC
       nginx.ingress.kubernetes.io/auth-type: "basic"
       nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
@@ -605,7 +705,7 @@ filer:
     timeoutSeconds: 10
 
   # secret env variables
-  secretExtraEnvironmentVars: []
+  secretExtraEnvironmentVars: {}
       # WEED_POSTGRES_USERNAME:
       #   secretKeyRef:
       #     name: postgres-credentials
@@ -679,11 +779,17 @@ s3:
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  # Labels to be added to the s3 pods
+  podLabels: {}
+
+  # Annotations to be added to the s3 pods
+  podAnnotations: {}
+
   # Resource requests, limits, etc. for the server cluster placement. This
   # should map directly to the value of the resources field for a PodSpec,
   # formatted as a multi-line string. By default no direct resource request
   # is made.
-  resources: {}
+  resources: null
 
   # Toleration Settings for server pods
   # This should be a multi-line string matching the Toleration array
@@ -706,6 +812,28 @@ s3:
   # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
   serviceAccountName: ""
 
+  # Configure security context for Pod
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # podSecurityContext:
+  #   enabled: true
+  #   runAsUser: 1000
+  #   runAsGroup: 3000
+  #   fsGroup: 2000
+  podSecurityContext: {}
+
+  # Configure security context for Container
+  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  # Example:
+  # containerSecurityContext:
+  #   enabled: true
+  #   runAsUser: 2000
+  #   allowPrivilegeEscalation: false
+  containerSecurityContext: {}
+
+  # You can also use emptyDir storage:
+  #  logs:
+  #    type: "emptyDir"
   logs:
     type: "hostPath"
     size: ""
@@ -746,7 +874,7 @@ s3:
     # host: false for "*" hostname
     host: "seaweedfs.cluster.local"
     # additional ingress annotations for the s3 endpoint
-    annotations: []
+    annotations: {}
     tls: []
 
 certificates:

other/java/client/src/main/proto/filer.proto

@@ -364,6 +364,7 @@ message LogEntry {
     int64 ts_ns = 1;
     int32 partition_key_hash = 2;
     bytes data = 3;
+    bytes key = 4;
 }
 
 message KeepConnectedRequest {
@@ -452,8 +453,9 @@ message LockRequest {
 }
 message LockResponse {
     string renew_token = 1;
-    string moved_to = 2;
-    string error = 3;
+    string lock_owner = 2;
+    string lock_host_moved_to = 3;
+    string error = 4;
 }
 message UnlockRequest {
     string name = 1;

unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go

@@ -66,6 +66,7 @@ func main() {
 
 	go func() {
 		ticker := time.NewTicker(1000 * time.Millisecond)
+		defer ticker.Stop()
 
 		var lastTime time.Time
 		var counter, size int64

unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go

@@ -72,6 +72,7 @@ func main() {
 
 	go func() {
 		ticker := time.NewTicker(500 * time.Millisecond)
+		defer ticker.Stop()
 
 		var lastTime time.Time
 		var counter, size int64

weed/cluster/lock_client.go

@@ -32,107 +32,104 @@ type LiveLock struct {
 	key            string
 	renewToken     string
 	expireAtNs     int64
-	filer          pb.ServerAddress
+	hostFiler      pb.ServerAddress
 	cancelCh       chan struct{}
 	grpcDialOption grpc.DialOption
 	isLocked       bool
-	owner          string
+	self           string
 	lc             *LockClient
+	owner          string
 }
 
-// NewLock creates a lock with a very long duration
-func (lc *LockClient) NewLock(key string, owner string) (lock *LiveLock) {
-	return lc.doNewLock(key, lock_manager.MaxDuration, owner)
-}
-
-// StartLock starts a goroutine to lock the key and returns immediately.
-func (lc *LockClient) StartLock(key string, owner string) (lock *LiveLock) {
+// NewShortLivedLock creates a lock with a 5-second duration
+func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLock) {
 	lock = &LiveLock{
 		key:            key,
-		filer:          lc.seedFiler,
+		hostFiler:      lc.seedFiler,
 		cancelCh:       make(chan struct{}),
-		expireAtNs:     time.Now().Add(lock_manager.MaxDuration).UnixNano(),
+		expireAtNs:     time.Now().Add(5 * time.Second).UnixNano(),
 		grpcDialOption: lc.grpcDialOption,
-		owner:          owner,
+		self:           owner,
+		lc:             lc,
+	}
+	lock.retryUntilLocked(5 * time.Second)
+	return
+}
+
+// StartLongLivedLock starts a goroutine to lock the key and returns immediately.
+func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerChange func(newLockOwner string)) (lock *LiveLock) {
+	lock = &LiveLock{
+		key:            key,
+		hostFiler:      lc.seedFiler,
+		cancelCh:       make(chan struct{}),
+		expireAtNs:     time.Now().Add(lock_manager.LiveLockTTL).UnixNano(),
+		grpcDialOption: lc.grpcDialOption,
+		self:           owner,
 		lc:             lc,
 	}
 	go func() {
-		util.RetryUntil("create lock:"+key, func() error {
-			errorMessage, err := lock.doLock(lock_manager.MaxDuration)
-			if err != nil {
-				glog.V(0).Infof("create lock %s: %s", key, err)
-				time.Sleep(time.Second)
-				return err
+		isLocked := false
+		lockOwner := ""
+		for {
+			if isLocked {
+				if err := lock.AttemptToLock(lock_manager.LiveLockTTL); err != nil {
+					glog.V(0).Infof("Lost lock %s: %v", key, err)
+					isLocked = false
+				}
+			} else {
+				if err := lock.AttemptToLock(lock_manager.LiveLockTTL); err == nil {
+					isLocked = true
+				}
 			}
-			if errorMessage != "" {
-				glog.V(4).Infof("create lock %s: %s", key, errorMessage)
-				time.Sleep(time.Second)
-				return fmt.Errorf("%v", errorMessage)
+			if lockOwner != lock.LockOwner() && lock.LockOwner() != "" {
+				glog.V(0).Infof("Lock owner changed from %s to %s", lockOwner, lock.LockOwner())
+				onLockOwnerChange(lock.LockOwner())
+				lockOwner = lock.LockOwner()
 			}
-			lock.isLocked = true
-			return nil
-		}, func(err error) (shouldContinue bool) {
-			if err != nil {
-				time.Sleep(time.Second)
+			select {
+			case <-lock.cancelCh:
+				return
+			default:
+				time.Sleep(lock_manager.RenewInterval)
 			}
-			return lock.renewToken == ""
-		})
-		lc.keepLock(lock)
+		}
 	}()
 	return
 }
 
-func (lc *LockClient) doNewLock(key string, lockDuration time.Duration, owner string) (lock *LiveLock) {
-	lock = &LiveLock{
-		key:            key,
-		filer:          lc.seedFiler,
-		cancelCh:       make(chan struct{}),
-		expireAtNs:     time.Now().Add(lockDuration).UnixNano(),
-		grpcDialOption: lc.grpcDialOption,
-		owner:          owner,
-		lc:             lc,
-	}
-	var needRenewal bool
-	if lockDuration > lc.maxLockDuration {
-		lockDuration = lc.maxLockDuration
-		needRenewal = true
-	}
-	util.RetryUntil("create lock:"+key, func() error {
-		errorMessage, err := lock.doLock(lockDuration)
-		if err != nil {
-			time.Sleep(time.Second)
-			return err
-		}
-		if errorMessage != "" {
-			time.Sleep(time.Second)
-			return fmt.Errorf("%v", errorMessage)
-		}
-		lock.isLocked = true
-		return nil
+func (lock *LiveLock) retryUntilLocked(lockDuration time.Duration) {
+	util.RetryUntil("create lock:"+lock.key, func() error {
+		return lock.AttemptToLock(lockDuration)
 	}, func(err error) (shouldContinue bool) {
 		if err != nil {
-			glog.Warningf("create lock %s: %s", key, err)
+			glog.Warningf("create lock %s: %s", lock.key, err)
 		}
 		return lock.renewToken == ""
 	})
+}
 
-	if needRenewal {
-		go lc.keepLock(lock)
+func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {
+	errorMessage, err := lock.doLock(lockDuration)
+	if err != nil {
+		time.Sleep(time.Second)
+		return err
 	}
-
-	return
+	if errorMessage != "" {
+		time.Sleep(time.Second)
+		return fmt.Errorf("%v", errorMessage)
+	}
+	lock.isLocked = true
+	return nil
 }
 
-func (lock *LiveLock) IsLocked() bool {
-	return lock.isLocked
-}
-
-func (lock *LiveLock) StopLock() error {
-	close(lock.cancelCh)
+func (lock *LiveLock) StopShortLivedLock() error {
 	if !lock.isLocked {
 		return nil
 	}
-	return pb.WithFilerClient(false, 0, lock.filer, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	defer func() {
+		lock.isLocked = false
+	}()
+	return pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		_, err := client.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
 			Name:       lock.key,
 			RenewToken: lock.renewToken,
@@ -141,69 +138,40 @@ func (lock *LiveLock) StopLock() error {
 	})
 }
 
-func (lc *LockClient) keepLock(lock *LiveLock) {
-	ticker := time.Tick(lc.sleepDuration)
-	for {
-		select {
-		case <-ticker:
-			// renew the lock if lock.expireAtNs is still greater than now
-			util.RetryUntil("keep lock:"+lock.key, func() error {
-				lockDuration := time.Duration(lock.expireAtNs-time.Now().UnixNano()) * time.Nanosecond
-				if lockDuration > lc.maxLockDuration {
-					lockDuration = lc.maxLockDuration
-				}
-				if lockDuration <= 0 {
-					return nil
-				}
-
-				errorMessage, err := lock.doLock(lockDuration)
-				if err != nil {
-					lock.isLocked = false
-					time.Sleep(time.Second)
-					return err
-				}
-				if errorMessage != "" {
-					lock.isLocked = false
-					time.Sleep(time.Second)
-					return fmt.Errorf("%v", errorMessage)
-				}
-				return nil
-			}, func(err error) (shouldContinue bool) {
-				if err == nil {
-					return false
-				}
-				glog.Warningf("keep lock %s: %v", lock.key, err)
-				return true
-			})
-			if !lock.isLocked {
-				return
-			}
-		case <-lock.cancelCh:
-			return
-		}
-	}
-}
-
 func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, err error) {
-	err = pb.WithFilerClient(false, 0, lock.filer, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		resp, err := client.DistributedLock(context.Background(), &filer_pb.LockRequest{
 			Name:          lock.key,
 			SecondsToLock: int64(lockDuration.Seconds()),
 			RenewToken:    lock.renewToken,
 			IsMoved:       false,
-			Owner:         lock.owner,
+			Owner:         lock.self,
 		})
-		if err == nil {
+		if err == nil && resp != nil {
 			lock.renewToken = resp.RenewToken
+		} else {
+			//this can be retried. Need to remember the last valid renewToken
+			lock.renewToken = ""
 		}
 		if resp != nil {
 			errorMessage = resp.Error
-			if resp.MovedTo != "" {
-				lock.filer = pb.ServerAddress(resp.MovedTo)
-				lock.lc.seedFiler = lock.filer
+			if resp.LockHostMovedTo != "" {
+				lock.hostFiler = pb.ServerAddress(resp.LockHostMovedTo)
+				lock.lc.seedFiler = lock.hostFiler
+			}
+			if resp.LockOwner != "" {
+				lock.owner = resp.LockOwner
+				// fmt.Printf("lock %s owner: %s\n", lock.key, lock.owner)
+			} else {
+				// fmt.Printf("lock %s has no owner\n", lock.key)
+				lock.owner = ""
 			}
 		}
 		return err
 	})
 	return
 }
+
+func (lock *LiveLock) LockOwner() string {
+	return lock.owner
+}

weed/cluster/lock_manager/distributed_lock_manager.go

@@ -2,11 +2,13 @@ package lock_manager
 
 import (
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"time"
 )
 
-const MaxDuration = 1<<63 - 1
+const RenewInterval = time.Second * 3
+const LiveLockTTL = time.Second * 7
 
 var NoLockServerError = fmt.Errorf("no lock server found")
 
@@ -24,7 +26,7 @@ func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
 	}
 }
 
-func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (renewToken string, movedTo pb.ServerAddress, err error) {
+func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (lockOwner string, renewToken string, movedTo pb.ServerAddress, err error) {
 	movedTo, err = dlm.findLockOwningFiler(key)
 	if err != nil {
 		return
@@ -32,7 +34,7 @@ func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64
 	if movedTo != dlm.Host {
 		return
 	}
-	renewToken, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
+	lockOwner, renewToken, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
 	return
 }
 
@@ -53,6 +55,8 @@ func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, move
 		return
 	}
 	if movedTo != dlm.Host {
+		servers := dlm.LockRing.GetSnapshot()
+		glog.V(0).Infof("lock %s not on current %s but on %s from %v", key, dlm.Host, movedTo, servers)
 		return
 	}
 	owner, err = dlm.lockManager.GetLockOwner(key)

weed/cluster/lock_manager/lock_manager.go

@@ -3,7 +3,8 @@ package lock_manager
 import (
 	"fmt"
 	"github.com/google/uuid"
-	"github.com/puzpuzpuz/xsync/v2"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"sync"
 	"time"
 )
 
@@ -11,10 +12,12 @@ var LockErrorNonEmptyTokenOnNewLock = fmt.Errorf("lock: non-empty token on a new
 var LockErrorNonEmptyTokenOnExpiredLock = fmt.Errorf("lock: non-empty token on an expired lock")
 var LockErrorTokenMismatch = fmt.Errorf("lock: token mismatch")
 var UnlockErrorTokenMismatch = fmt.Errorf("unlock: token mismatch")
+var LockNotFound = fmt.Errorf("lock not found")
 
 // LockManager local lock manager, used by distributed lock manager
 type LockManager struct {
-	locks *xsync.MapOf[string, *Lock]
+	locks      map[string]*Lock
+	accessLock sync.RWMutex
 }
 type Lock struct {
 	Token       string
@@ -25,126 +28,155 @@ type Lock struct {
 
 func NewLockManager() *LockManager {
 	t := &LockManager{
-		locks: xsync.NewMapOf[*Lock](),
+		locks: make(map[string]*Lock),
 	}
 	go t.CleanUp()
 	return t
 }
 
-func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner string) (renewToken string, err error) {
-	lm.locks.Compute(path, func(oldValue *Lock, loaded bool) (newValue *Lock, delete bool) {
-		if oldValue != nil {
-			if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < time.Now().UnixNano() {
-				// lock is expired, set to a new lock
-				if token != "" {
-					err = LockErrorNonEmptyTokenOnExpiredLock
-					return nil, false
-				} else {
-					// new lock
-					renewToken = uuid.New().String()
-					return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}, false
-				}
-			}
-			// not expired
-			if oldValue.Token == token {
-				// token matches, renew the lock
-				renewToken = uuid.New().String()
-				return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}, false
+func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner string) (lockOwner, renewToken string, err error) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	glog.V(4).Infof("lock %s %v %v %v", path, time.Unix(0, expiredAtNs), token, owner)
+
+	if oldValue, found := lm.locks[path]; found {
+		if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < time.Now().UnixNano() {
+			// lock is expired, set to a new lock
+			if token != "" {
+				glog.V(4).Infof("lock expired key %s non-empty token %v owner %v ts %s", path, token, owner, time.Unix(0, oldValue.ExpiredAtNs))
+				err = LockErrorNonEmptyTokenOnExpiredLock
+				return
 			} else {
-				err = LockErrorTokenMismatch
-				return oldValue, false
+				// new lock
+				renewToken = uuid.New().String()
+				glog.V(4).Infof("key %s new token %v owner %v", path, renewToken, owner)
+				lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
+				return
 			}
+		}
+		// not expired
+		lockOwner = oldValue.Owner
+		if oldValue.Token == token {
+			// token matches, renew the lock
+			renewToken = uuid.New().String()
+			glog.V(4).Infof("key %s old token %v owner %v => %v owner %v", path, oldValue.Token, oldValue.Owner, renewToken, owner)
+			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
+			return
 		} else {
 			if token == "" {
 				// new lock
-				renewToken = uuid.New().String()
-				return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}, false
-			} else {
-				err = LockErrorNonEmptyTokenOnNewLock
-				return nil, false
+				glog.V(4).Infof("key %s locked by %v", path, oldValue.Owner)
+				err = fmt.Errorf("lock already owned by %v", oldValue.Owner)
+				return
 			}
+			glog.V(4).Infof("key %s expected token %v owner %v received %v from %v", path, oldValue.Token, oldValue.Owner, token, owner)
+			err = fmt.Errorf("lock: token mismatch")
+			return
 		}
-	})
-	return
+	} else {
+		glog.V(4).Infof("key %s no lock owner %v", path, owner)
+		if token == "" {
+			// new lock
+			glog.V(4).Infof("key %s new token %v owner %v", path, token, owner)
+			renewToken = uuid.New().String()
+			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
+			return
+		} else {
+			glog.V(4).Infof("key %s non-empty token %v owner %v", path, token, owner)
+			err = LockErrorNonEmptyTokenOnNewLock
+			return
+		}
+	}
 }
 
 func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, err error) {
-	lm.locks.Compute(path, func(oldValue *Lock, loaded bool) (newValue *Lock, delete bool) {
-		if oldValue != nil {
-			now := time.Now()
-			if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < now.UnixNano() {
-				// lock is expired, delete it
-				isUnlocked = true
-				return nil, true
-			}
-			if oldValue.Token == token {
-				if oldValue.ExpiredAtNs <= now.UnixNano() {
-					isUnlocked = true
-					return nil, true
-				}
-				return oldValue, false
-			} else {
-				isUnlocked = false
-				err = UnlockErrorTokenMismatch
-				return oldValue, false
-			}
-		} else {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if oldValue, found := lm.locks[path]; found {
+		now := time.Now()
+		if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < now.UnixNano() {
+			// lock is expired, delete it
 			isUnlocked = true
-			return nil, true
+			glog.V(4).Infof("key %s expired at %v", path, time.Unix(0, oldValue.ExpiredAtNs))
+			delete(lm.locks, path)
+			return
 		}
-	})
+		if oldValue.Token == token {
+			isUnlocked = true
+			glog.V(4).Infof("key %s unlocked with %v", path, token)
+			delete(lm.locks, path)
+			return
+		} else {
+			isUnlocked = false
+			err = UnlockErrorTokenMismatch
+			return
+		}
+	}
+	err = LockNotFound
 	return
 }
 
 func (lm *LockManager) CleanUp() {
+
 	for {
 		time.Sleep(1 * time.Minute)
 		now := time.Now().UnixNano()
-		lm.locks.Range(func(key string, value *Lock) bool {
+
+		lm.accessLock.Lock()
+		for key, value := range lm.locks {
 			if value == nil {
-				return true
+				continue
 			}
 			if now > value.ExpiredAtNs {
-				lm.locks.Delete(key)
-				return true
+				glog.V(4).Infof("key %s expired at %v", key, time.Unix(0, value.ExpiredAtNs))
+				delete(lm.locks, key)
 			}
-			return true
-		})
+		}
+		lm.accessLock.Unlock()
 	}
 }
 
 // SelectLocks takes out locks by key
 // if keyFn return true, the lock will be taken out
 func (lm *LockManager) SelectLocks(selectFn func(key string) bool) (locks []*Lock) {
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
+
 	now := time.Now().UnixNano()
-	lm.locks.Range(func(key string, lock *Lock) bool {
+
+	for key, lock := range lm.locks {
 		if now > lock.ExpiredAtNs {
-			lm.locks.Delete(key)
-			return true
+			glog.V(4).Infof("key %s expired at %v", key, time.Unix(0, lock.ExpiredAtNs))
+			delete(lm.locks, key)
+			continue
 		}
 		if selectFn(key) {
-			lm.locks.Delete(key)
+			glog.V(4).Infof("key %s selected and deleted", key)
+			delete(lm.locks, key)
 			lock.Key = key
 			locks = append(locks, lock)
 		}
-		return true
-	})
+	}
 	return
 }
 
 // InsertLock inserts a lock unconditionally
 func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string, owner string) {
-	lm.locks.Store(path, &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner})
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner}
 }
 
 func (lm *LockManager) GetLockOwner(key string) (owner string, err error) {
-	lm.locks.Range(func(k string, lock *Lock) bool {
-		if k == key && lock != nil {
-			owner = lock.Owner
-			return false
-		}
-		return true
-	})
-	return
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
 
+	if lock, found := lm.locks[key]; found {
+		return lock.Owner, nil
+	}
+	err = LockNotFound
+	return
 }

weed/cluster/lock_manager/lock_ring.go

@@ -1,6 +1,7 @@
 package lock_manager
 
 import (
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"sort"
@@ -39,9 +40,11 @@ func (r *LockRing) SetTakeSnapshotCallback(onTakeSnapshot func(snapshot []pb.Ser
 // AddServer adds a server to the ring
 // if the previous snapshot passed the snapshot interval, create a new snapshot
 func (r *LockRing) AddServer(server pb.ServerAddress) {
+	glog.V(0).Infof("add server %v", server)
 	r.Lock()
 
 	if _, found := r.candidateServers[server]; found {
+		glog.V(0).Infof("add server: already exists %v", server)
 		r.Unlock()
 		return
 	}
@@ -53,6 +56,8 @@ func (r *LockRing) AddServer(server pb.ServerAddress) {
 }
 
 func (r *LockRing) RemoveServer(server pb.ServerAddress) {
+	glog.V(0).Infof("remove server %v", server)
+
 	r.Lock()
 
 	if _, found := r.candidateServers[server]; !found {
@@ -72,7 +77,13 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
 		return servers[i] < servers[j]
 	})
 
+	r.Lock()
 	r.lastUpdateTime = time.Now()
+	// init candidateServers
+	for _, server := range servers {
+		r.candidateServers[server] = struct{}{}
+	}
+	r.Unlock()
 
 	r.addOneSnapshot(servers)
 

weed/command/compact.go

@@ -28,7 +28,7 @@ var (
 	compactVolumePath        = cmdCompact.Flag.String("dir", ".", "data directory to store files")
 	compactVolumeCollection  = cmdCompact.Flag.String("collection", "", "volume collection name")
 	compactVolumeId          = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.")
-	compactMethod            = cmdCompact.Flag.Int("method", 0, "option to choose which compact method. use 0 or 1.")
+	compactMethod            = cmdCompact.Flag.Int("method", 0, "option to choose which compact method. use 0 (default) or 1.")
 	compactVolumePreallocate = cmdCompact.Flag.Int64("preallocateMB", 0, "preallocate volume disk space")
 )
 

weed/command/filer.go

@@ -62,6 +62,7 @@ type FilerOptions struct {
 	downloadMaxMBps         *int
 	diskType                *string
 	allowedOrigins          *string
+	exposeDirectoryData     *bool
 }
 
 func init() {
@@ -93,6 +94,7 @@ func init() {
 	f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
 	f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
 	f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins")
+	f.exposeDirectoryData = cmdFiler.Flag.Bool("exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI")
 
 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")

weed/command/filer_backup.go

@@ -85,8 +85,7 @@ const (
 func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions, clientId int32, clientEpoch int32) error {
 
 	// find data sink
-	config := util.GetViper()
-	dataSink := findSink(config)
+	dataSink := findSink(util.GetViper())
 	if dataSink == nil {
 		return fmt.Errorf("no data sink configured in replication.toml")
 	}

weed/command/filer_sync.go

@@ -394,7 +394,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			glog.V(0).Infof("received %v", resp)
 		}
 
-		if isMultipartUploadDir(resp.Directory) {
+		if isMultipartUploadDir(resp.Directory + "/") {
 			return nil
 		}
 

weed/command/fix.go

@@ -32,12 +32,15 @@ var cmdFix = &Command{
 var (
 	fixVolumeCollection = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed")
 	fixVolumeId         = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed")
+	fixIncludeDeleted   = cmdFix.Flag.Bool("includeDeleted", true, "include deleted entries in the index file")
 	fixIgnoreError      = cmdFix.Flag.Bool("ignoreError", false, "an optional, if true will be processed despite errors")
 )
 
 type VolumeFileScanner4Fix struct {
-	version needle.Version
-	nm      *needle_map.MemDb
+	version        needle.Version
+	nm             *needle_map.MemDb
+	nmDeleted      *needle_map.MemDb
+	includeDeleted bool
 }
 
 func (scanner *VolumeFileScanner4Fix) VisitSuperBlock(superBlock super_block.SuperBlock) error {
@@ -50,13 +53,20 @@ func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
 }
 
 func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
-	glog.V(2).Infof("key %d offset %d size %d disk_size %d compressed %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed())
+	glog.V(2).Infof("key %v offset %d size %d disk_size %d compressed %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed())
 	if n.Size.IsValid() {
-		pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size)
-		glog.V(2).Infof("saved %d with error %v", n.Size, pe)
+		if pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size); pe != nil {
+			return fmt.Errorf("saved %d with error %v", n.Size, pe)
+		}
 	} else {
-		glog.V(2).Infof("skipping deleted file ...")
-		return scanner.nm.Delete(n.Id)
+		if scanner.includeDeleted {
+			if pe := scanner.nmDeleted.Set(n.Id, types.ToOffset(offset), types.TombstoneFileSize); pe != nil {
+				return fmt.Errorf("saved deleted %d with error %v", n.Size, pe)
+			}
+		} else {
+			glog.V(2).Infof("skipping deleted file ...")
+			return scanner.nm.Delete(n.Id)
+		}
 	}
 	return nil
 }
@@ -109,21 +119,45 @@ func runFix(cmd *Command, args []string) bool {
 			if *fixVolumeId != 0 && *fixVolumeId != volumeId {
 				continue
 			}
-			doFixOneVolume(basePath, baseFileName, collection, volumeId)
+			doFixOneVolume(basePath, baseFileName, collection, volumeId, *fixIncludeDeleted)
 		}
 	}
 	return true
 }
 
-func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64) {
+func SaveToIdx(scaner *VolumeFileScanner4Fix, idxName string) (ret error) {
+	idxFile, err := os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return
+	}
+	defer func() {
+		idxFile.Close()
+	}()
+
+	return scaner.nm.AscendingVisit(func(value needle_map.NeedleValue) error {
+		_, err := idxFile.Write(value.ToBytes())
+		if scaner.includeDeleted && err == nil {
+			if deleted, ok := scaner.nmDeleted.Get(value.Key); ok {
+				_, err = idxFile.Write(deleted.ToBytes())
+			}
+		}
+		return err
+	})
+}
+
+func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64, fixIncludeDeleted bool) {
 	indexFileName := path.Join(basepath, baseFileName+".idx")
 
 	nm := needle_map.NewMemDb()
+	nmDeleted := needle_map.NewMemDb()
 	defer nm.Close()
+	defer nmDeleted.Close()
 
 	vid := needle.VolumeId(volumeId)
 	scanner := &VolumeFileScanner4Fix{
-		nm: nm,
+		nm:             nm,
+		nmDeleted:      nmDeleted,
+		includeDeleted: fixIncludeDeleted,
 	}
 
 	if err := storage.ScanVolumeFile(basepath, collection, vid, storage.NeedleMapInMemory, scanner); err != nil {
@@ -135,12 +169,12 @@ func doFixOneVolume(basepath string, baseFileName string, collection string, vol
 		}
 	}
 
-	if err := nm.SaveToIdx(indexFileName); err != nil {
-		os.Remove(indexFileName)
+	if err := SaveToIdx(scanner, indexFileName); err != nil {
 		err := fmt.Errorf("save to .idx File: %v", err)
 		if *fixIgnoreError {
 			glog.Error(err)
 		} else {
+			os.Remove(indexFileName)
 			glog.Fatal(err)
 		}
 	}

weed/command/master_follower.go

@@ -52,7 +52,7 @@ var cmdMasterFollower = &Command{
 	In most cases, the master follower is not needed. In big data centers with thousands of volume
 	servers. In theory, the master may have trouble to keep up with the write requests and read requests.
 
-	The master follower can relieve the master from from read requests, which only needs to
+	The master follower can relieve the master from read requests, which only needs to
 	lookup a fileId or volumeId.
 
 	The master follower currently can handle fileId lookup requests:

weed/command/mount_std.go

@@ -201,7 +201,9 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 			ioSizeMB *= 2
 		}
 		fuseMountOptions.Options = append(fuseMountOptions.Options, "daemon_timeout=600")
-		fuseMountOptions.Options = append(fuseMountOptions.Options, "noapplexattr")
+		if runtime.GOARCH == "amd64" {
+			fuseMountOptions.Options = append(fuseMountOptions.Options, "noapplexattr")
+		}
 		// fuseMountOptions.Options = append(fuseMountOptions.Options, "novncache") // need to test effectiveness
 		fuseMountOptions.Options = append(fuseMountOptions.Options, "slow_statfs")
 		fuseMountOptions.Options = append(fuseMountOptions.Options, "volname="+serverFriendlyName)

weed/command/scaffold/filer.toml

@@ -270,10 +270,23 @@ username = ""
 password = ""
 key_prefix = "seaweedfs."
 timeout = "3s"
+# Set the CA certificate path
+tls_ca_file=""
+# Set the client certificate path
+tls_client_crt_file=""
+# Set the client private key path
+tls_client_key_file=""
 
 [mongodb]
 enabled = false
 uri = "mongodb://localhost:27017"
+username = ""
+password = ""
+ssl = false
+ssl_ca_file = ""
+ssl_cert_file = ""
+ssl_key_file = "
+insecure_skip_verify = false
 option_pool_size = 0
 database = "seaweedfs"
 

weed/command/scaffold/security.toml

@@ -24,6 +24,13 @@ expires_after_seconds = 10           # seconds
 [access]
 ui = false
 
+# by default the filer UI is enabled. This can be a security risk if the filer is exposed to the public
+# and the JWT for reads is not set. If you don't want the public to have access to the objects in your
+# storage, and you haven't set the JWT for reads it is wise to disable access to directory metadata.
+# This disables access to the Filer UI, and will no longer return directory metadata in GET requests.
+[filer.expose_directory_metadata]
+enabled = true
+
 # this jwt signing key is read by master and volume server, and it is used for read operations:
 # - the Master server generates the JWT, which can be used to read a certain file on a volume server
 # - the Volume server validates the JWT on reading

weed/command/server.go

@@ -118,6 +118,7 @@ func init() {
 	filerOptions.showUIDirectoryDelete = cmdServer.Flag.Bool("filer.ui.deleteDir", true, "enable filer UI show delete directory button")
 	filerOptions.downloadMaxMBps = cmdServer.Flag.Int("filer.downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
 	filerOptions.diskType = cmdServer.Flag.String("filer.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
+	filerOptions.exposeDirectoryData = cmdServer.Flag.Bool("filer.exposeDirectoryData", true, "expose directory data via filer. If false, filer UI will be innaccessible.")
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")

weed/command/shell.go

@@ -19,7 +19,7 @@ func init() {
 	cmdShell.Run = runShell // break init cycle
 	shellOptions.Masters = cmdShell.Flag.String("master", "", "comma-separated master servers, e.g. localhost:9333")
 	shellOptions.FilerGroup = cmdShell.Flag.String("filerGroup", "", "filerGroup for the filers")
-	shellInitialFiler = cmdShell.Flag.String("filer", "", "filer host and port, e.g. localhost:8888")
+	shellInitialFiler = cmdShell.Flag.String("filer", "", "filer host and port for initial connection, e.g. localhost:8888")
 	shellCluster = cmdShell.Flag.String("cluster", "", "cluster defined in shell.toml")
 }
 
@@ -30,32 +30,36 @@ var cmdShell = &Command{
 
 	Generate shell.toml via "weed scaffold -config=shell"
 
-  `,
+`,
 }
 
 func runShell(command *Command, args []string) bool {
 
 	util.LoadConfiguration("security", false)
 	shellOptions.GrpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
+	shellOptions.Directory = "/"
+
+	util.LoadConfiguration("shell", false)
+	viper := util.GetViper()
+	cluster := viper.GetString("cluster.default")
+	if *shellCluster != "" {
+		cluster = *shellCluster
+	}
 
 	if *shellOptions.Masters == "" {
-		util.LoadConfiguration("shell", false)
-		v := util.GetViper()
-		cluster := v.GetString("cluster.default")
-		if *shellCluster != "" {
-			cluster = *shellCluster
-		}
 		if cluster == "" {
 			*shellOptions.Masters = "localhost:9333"
 		} else {
-			*shellOptions.Masters = v.GetString("cluster." + cluster + ".master")
-			*shellInitialFiler = v.GetString("cluster." + cluster + ".filer")
-			fmt.Printf("master: %s filer: %s\n", *shellOptions.Masters, *shellInitialFiler)
+			*shellOptions.Masters = viper.GetString("cluster." + cluster + ".master")
 		}
 	}
 
-	shellOptions.FilerAddress = pb.ServerAddress(*shellInitialFiler)
-	shellOptions.Directory = "/"
+	filerAddress := *shellInitialFiler
+	if filerAddress == "" && cluster != "" {
+		filerAddress = viper.GetString("cluster." + cluster + ".filer")
+	}
+	shellOptions.FilerAddress = pb.ServerAddress(filerAddress)
+	fmt.Printf("master: %s filer: %s\n", *shellOptions.Masters, shellOptions.FilerAddress)
 
 	shell.RunShell(shellOptions)
 

weed/filer/etcd/etcd_store.go

@@ -1,9 +1,10 @@
 package etcd
 
 import (
-	"bytes"
 	"context"
+	"crypto/tls"
 	"fmt"
+	"go.etcd.io/etcd/client/pkg/v3/transport"
 	"strings"
 	"time"
 
@@ -26,49 +27,78 @@ func init() {
 type EtcdStore struct {
 	client        *clientv3.Client
 	etcdKeyPrefix string
+	timeout       time.Duration
 }
 
 func (store *EtcdStore) GetName() string {
 	return "etcd"
 }
 
-func (store *EtcdStore) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
+func (store *EtcdStore) Initialize(configuration weed_util.Configuration, prefix string) error {
+	configuration.SetDefault(prefix+"servers", "localhost:2379")
+	configuration.SetDefault(prefix+"timeout", "3s")
+
 	servers := configuration.GetString(prefix + "servers")
-	if servers == "" {
-		servers = "localhost:2379"
-	}
 
 	username := configuration.GetString(prefix + "username")
 	password := configuration.GetString(prefix + "password")
 	store.etcdKeyPrefix = configuration.GetString(prefix + "key_prefix")
 
-	timeout := configuration.GetString(prefix + "timeout")
-	if timeout == "" {
-		timeout = "3s"
+	timeoutStr := configuration.GetString(prefix + "timeout")
+	timeout, err := time.ParseDuration(timeoutStr)
+	if err != nil {
+		return fmt.Errorf("parse etcd store timeout: %v", err)
+	}
+	store.timeout = timeout
+
+	certFile := configuration.GetString(prefix + "tls_client_crt_file")
+	keyFile := configuration.GetString(prefix + "tls_client_key_file")
+	caFile := configuration.GetString(prefix + "tls_ca_file")
+
+	var tlsConfig *tls.Config
+	if caFile != "" {
+		tlsInfo := transport.TLSInfo{
+			CertFile:      certFile,
+			KeyFile:       keyFile,
+			TrustedCAFile: caFile,
+		}
+		var err error
+		tlsConfig, err = tlsInfo.ClientConfig()
+		if err != nil {
+			return fmt.Errorf("TLS client configuration error: %v", err)
+		}
 	}
 
-	return store.initialize(servers, username, password, timeout)
+	return store.initialize(servers, username, password, store.timeout, tlsConfig)
 }
 
-func (store *EtcdStore) initialize(servers string, username string, password string, timeout string) (err error) {
+func (store *EtcdStore) initialize(servers, username, password string, timeout time.Duration, tlsConfig *tls.Config) error {
 	glog.Infof("filer store etcd: %s", servers)
 
-	to, err := time.ParseDuration(timeout)
-	if err != nil {
-		return fmt.Errorf("parse timeout %s: %s", timeout, err)
-	}
-
-	store.client, err = clientv3.New(clientv3.Config{
+	client, err := clientv3.New(clientv3.Config{
 		Endpoints:   strings.Split(servers, ","),
 		Username:    username,
 		Password:    password,
-		DialTimeout: to,
+		DialTimeout: timeout,
+		TLS:         tlsConfig,
 	})
 	if err != nil {
 		return fmt.Errorf("connect to etcd %s: %s", servers, err)
 	}
 
-	return
+	ctx, cancel := context.WithTimeout(context.Background(), store.timeout)
+	defer cancel()
+
+	resp, err := client.Status(ctx, client.Endpoints()[0])
+	if err != nil {
+		client.Close()
+		return fmt.Errorf("error checking etcd connection: %s", err)
+	}
+
+	glog.V(0).Infof("сonnection to etcd has been successfully verified. etcd version: %s", resp.Version)
+	store.client = client
+
+	return nil
 }
 
 func (store *EtcdStore) BeginTransaction(ctx context.Context) (context.Context, error) {
@@ -148,26 +178,20 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_
 }
 
 func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
-	return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
-}
-
-func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
-	directoryPrefix := genDirectoryKeyPrefix(dirPath, "")
+	directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix)
 	lastFileStart := directoryPrefix
 	if startFileName != "" {
 		lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
 	}
 
 	resp, err := store.client.Get(ctx, store.etcdKeyPrefix+string(lastFileStart),
-		clientv3.WithFromKey(), clientv3.WithLimit(limit+1))
+		clientv3.WithRange(clientv3.GetPrefixRangeEnd(store.etcdKeyPrefix+string(directoryPrefix))),
+		clientv3.WithLimit(limit+1))
 	if err != nil {
 		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
 	}
 
 	for _, kv := range resp.Kvs {
-		if !bytes.HasPrefix(kv.Key, directoryPrefix) {
-			break
-		}
 		fileName := getNameFromKey(kv.Key)
 		if fileName == "" {
 			continue
@@ -196,6 +220,10 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u
 	return lastFileName, err
 }
 
+func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
+}
+
 func genKey(dirPath, fileName string) (key []byte) {
 	key = []byte(dirPath)
 	key = append(key, DIR_FILE_SEPARATOR)

weed/filer/etcd/etcd_store_test.go

@@ -10,7 +10,7 @@ func TestStore(t *testing.T) {
 	// to set up local env
 	if false {
 		store := &EtcdStore{}
-		store.initialize("localhost:2379", "", "", "3s")
+		store.initialize("localhost:2379", "", "", 3, nil)
 		store_test.TestFilerStore(t, store)
 	}
 }

weed/filer/filechunk_manifest.go

@@ -5,8 +5,6 @@ import (
 	"fmt"
 	"io"
 	"math"
-	"net/url"
-	"strings"
 	"sync"
 	"time"
 
@@ -109,7 +107,7 @@ func fetchWholeChunk(bytesBuffer *bytes.Buffer, lookupFileIdFn wdclient.LookupFi
 		glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err)
 		return err
 	}
-	err = retriedStreamFetchChunkData(bytesBuffer, urlStrings, cipherKey, isGzipped, true, 0, 0)
+	err = retriedStreamFetchChunkData(bytesBuffer, urlStrings, "", cipherKey, isGzipped, true, 0, 0)
 	if err != nil {
 		return err
 	}
@@ -122,47 +120,10 @@ func fetchChunkRange(buffer []byte, lookupFileIdFn wdclient.LookupFileIdFunction
 		glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err)
 		return 0, err
 	}
-	return retriedFetchChunkData(buffer, urlStrings, cipherKey, isGzipped, false, offset)
+	return util.RetriedFetchChunkData(buffer, urlStrings, cipherKey, isGzipped, false, offset)
 }
 
-func retriedFetchChunkData(buffer []byte, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64) (n int, err error) {
-
-	var shouldRetry bool
-
-	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
-		for _, urlString := range urlStrings {
-			n = 0
-			if strings.Contains(urlString, "%") {
-				urlString = url.PathEscape(urlString)
-			}
-			shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, len(buffer), func(data []byte) {
-				if n < len(buffer) {
-					x := copy(buffer[n:], data)
-					n += x
-				}
-			})
-			if !shouldRetry {
-				break
-			}
-			if err != nil {
-				glog.V(0).Infof("read %s failed, err: %v", urlString, err)
-			} else {
-				break
-			}
-		}
-		if err != nil && shouldRetry {
-			glog.V(0).Infof("retry reading in %v", waitTime)
-			time.Sleep(waitTime)
-		} else {
-			break
-		}
-	}
-
-	return n, err
-
-}
-
-func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) {
+func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, jwt string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) {
 
 	var shouldRetry bool
 	var totalWritten int
@@ -171,7 +132,7 @@ func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, cipherKe
 		for _, urlString := range urlStrings {
 			var localProcessed int
 			var writeErr error
-			shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
+			shouldRetry, err = util.ReadUrlAsStreamAuthenticated(urlString+"?readDeleted=true", jwt, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
 				if totalWritten > localProcessed {
 					toBeSkipped := totalWritten - localProcessed
 					if len(data) <= toBeSkipped {

weed/filer/filer.go

@@ -69,7 +69,7 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH
 		f.UniqueFilerId = -f.UniqueFilerId
 	}
 
-	f.LocalMetaLogBuffer = log_buffer.NewLogBuffer("local", LogFlushInterval, f.logFlushFunc, notifyFn)
+	f.LocalMetaLogBuffer = log_buffer.NewLogBuffer("local", LogFlushInterval, f.logFlushFunc, nil, notifyFn)
 	f.metaLogCollection = collection
 	f.metaLogReplication = replication
 
@@ -377,6 +377,6 @@ func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, sta
 }
 
 func (f *Filer) Shutdown() {
-	f.LocalMetaLogBuffer.Shutdown()
+	f.LocalMetaLogBuffer.ShutdownLogBuffer()
 	f.Store.Shutdown()
 }

weed/filer/filer_conf.go

@@ -27,7 +27,7 @@ const (
 )
 
 type FilerConf struct {
-	rules ptrie.Trie
+	rules ptrie.Trie[*filer_pb.FilerConf_PathConf]
 }
 
 func ReadFilerConf(filerGrpcAddress pb.ServerAddress, grpcDialOption grpc.DialOption, masterClient *wdclient.MasterClient) (*FilerConf, error) {
@@ -55,7 +55,7 @@ func ReadFilerConf(filerGrpcAddress pb.ServerAddress, grpcDialOption grpc.DialOp
 
 func NewFilerConf() (fc *FilerConf) {
 	fc = &FilerConf{
-		rules: ptrie.New(),
+		rules: ptrie.New[*filer_pb.FilerConf_PathConf](),
 	}
 	return fc
 }
@@ -120,8 +120,8 @@ func (fc *FilerConf) AddLocationConf(locConf *filer_pb.FilerConf_PathConf) (err
 }
 
 func (fc *FilerConf) DeleteLocationConf(locationPrefix string) {
-	rules := ptrie.New()
-	fc.rules.Walk(func(key []byte, value interface{}) bool {
+	rules := ptrie.New[*filer_pb.FilerConf_PathConf]()
+	fc.rules.Walk(func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
 		if string(key) == locationPrefix {
 			return true
 		}
@@ -135,9 +135,8 @@ func (fc *FilerConf) DeleteLocationConf(locationPrefix string) {
 
 func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf_PathConf) {
 	pathConf = &filer_pb.FilerConf_PathConf{}
-	fc.rules.MatchPrefix([]byte(path), func(key []byte, value interface{}) bool {
-		t := value.(*filer_pb.FilerConf_PathConf)
-		mergePathConf(pathConf, t)
+	fc.rules.MatchPrefix([]byte(path), func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
+		mergePathConf(pathConf, value)
 		return true
 	})
 	return pathConf
@@ -145,10 +144,9 @@ func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf
 
 func (fc *FilerConf) GetCollectionTtls(collection string) (ttls map[string]string) {
 	ttls = make(map[string]string)
-	fc.rules.Walk(func(key []byte, value interface{}) bool {
-		t := value.(*filer_pb.FilerConf_PathConf)
-		if t.Collection == collection {
-			ttls[t.LocationPrefix] = t.GetTtl()
+	fc.rules.Walk(func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
+		if value.Collection == collection {
+			ttls[value.LocationPrefix] = value.GetTtl()
 		}
 		return true
 	})
@@ -176,9 +174,8 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
 
 func (fc *FilerConf) ToProto() *filer_pb.FilerConf {
 	m := &filer_pb.FilerConf{}
-	fc.rules.Walk(func(key []byte, value interface{}) bool {
-		pathConf := value.(*filer_pb.FilerConf_PathConf)
-		m.Locations = append(m.Locations, pathConf)
+	fc.rules.Walk(func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
+		m.Locations = append(m.Locations, value)
 		return true
 	})
 	return m

weed/filer/filer_delete_entry.go

@@ -38,7 +38,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("delete directory %s: %v", p, err)
+			glog.V(2).Infof("delete directory %s: %v", p, err)
 			return fmt.Errorf("delete directory %s: %v", p, err)
 		}
 	}
@@ -76,7 +76,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 			}
 			if lastFileName == "" && !isRecursive && len(entries) > 0 {
 				// only for first iteration in the loop
-				glog.V(0).Infof("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name())
+				glog.V(2).Infof("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name())
 				return fmt.Errorf("%s: %s", MsgFailDelNonEmptyFolder, entry.FullPath)
 			}
 

weed/filer/filer_deletion.go

@@ -2,7 +2,6 @@ package filer
 
 import (
 	"github.com/seaweedfs/seaweedfs/weed/storage"
-	"math"
 	"strings"
 	"time"
 
@@ -138,47 +137,19 @@ func (f *Filer) DeleteChunksNotRecursive(chunks []*filer_pb.FileChunk) {
 }
 
 func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
-
-	if oldEntry == nil {
-		return
+	var oldChunks, newChunks []*filer_pb.FileChunk
+	if oldEntry != nil {
+		oldChunks = oldEntry.GetChunks()
 	}
-	if newEntry == nil {
-		f.DeleteChunks(oldEntry.GetChunks())
-		return
+	if newEntry != nil {
+		newChunks = newEntry.GetChunks()
 	}
 
-	var toDelete []*filer_pb.FileChunk
-	newChunkIds := make(map[string]bool)
-	newDataChunks, newManifestChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(),
-		newEntry.GetChunks(), 0, math.MaxInt64)
-	if err != nil {
-		glog.Errorf("Failed to resolve new entry chunks when delete old entry chunks. new: %s, old: %s",
-			newEntry.GetChunks(), oldEntry.Chunks)
-		return
-	}
-	for _, newChunk := range newDataChunks {
-		newChunkIds[newChunk.GetFileIdString()] = true
-	}
-	for _, newChunk := range newManifestChunks {
-		newChunkIds[newChunk.GetFileIdString()] = true
-	}
-
-	oldDataChunks, oldManifestChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(),
-		oldEntry.GetChunks(), 0, math.MaxInt64)
+	toDelete, err := MinusChunks(f.MasterClient.GetLookupFileIdFunction(), oldChunks, newChunks)
 	if err != nil {
 		glog.Errorf("Failed to resolve old entry chunks when delete old entry chunks. new: %s, old: %s",
-			newEntry.GetChunks(), oldEntry.GetChunks())
+			newChunks, oldChunks)
 		return
 	}
-	for _, oldChunk := range oldDataChunks {
-		if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found {
-			toDelete = append(toDelete, oldChunk)
-		}
-	}
-	for _, oldChunk := range oldManifestChunks {
-		if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found {
-			toDelete = append(toDelete, oldChunk)
-		}
-	}
 	f.DeleteChunksNotRecursive(toDelete)
 }

weed/filer/filer_notify.go

@@ -3,6 +3,7 @@ package filer
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
 	"io"
 	"math"
 	"regexp"
@@ -82,11 +83,11 @@ func (f *Filer) logMetaEvent(ctx context.Context, fullpath string, eventNotifica
 		return
 	}
 
-	f.LocalMetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
+	f.LocalMetaLogBuffer.AddDataToBuffer([]byte(dir), data, event.TsNs)
 
 }
 
-func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {
+func (f *Filer) logFlushFunc(logBuffer *log_buffer.LogBuffer, startTime, stopTime time.Time, buf []byte) {
 
 	if len(buf) == 0 {
 		return
@@ -113,11 +114,10 @@ var (
 	VolumeNotFoundPattern = regexp.MustCompile(`volume \d+? not found`)
 )
 
-func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, stopTsNs int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, isDone bool, err error) {
+func (f *Filer) ReadPersistedLogBuffer(startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastTsNs int64, isDone bool, err error) {
 
-	startTime = startTime.UTC()
-	startDate := fmt.Sprintf("%04d-%02d-%02d", startTime.Year(), startTime.Month(), startTime.Day())
-	startHourMinute := fmt.Sprintf("%02d-%02d", startTime.Hour(), startTime.Minute())
+	startDate := fmt.Sprintf("%04d-%02d-%02d", startPosition.Year(), startPosition.Month(), startPosition.Day())
+	startHourMinute := fmt.Sprintf("%02d-%02d", startPosition.Hour(), startPosition.Minute())
 	var stopDate, stopHourMinute string
 	if stopTsNs != 0 {
 		stopTime := time.Unix(0, stopTsNs+24*60*60*int64(time.Nanosecond)).UTC()
@@ -126,7 +126,7 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, stopTsNs int64, each
 	}
 
 	sizeBuf := make([]byte, 4)
-	startTsNs := startTime.UnixNano()
+	startTsNs := startPosition.UnixNano()
 
 	dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, math.MaxInt32, "", "", "")
 	if listDayErr != nil {
@@ -177,7 +177,7 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, stopTsNs int64, each
 	return lastTsNs, isDone, nil
 }
 
-func ReadEachLogEntry(r io.Reader, sizeBuf []byte, startTsNs, stopTsNs int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) {
+func ReadEachLogEntry(r io.Reader, sizeBuf []byte, startTsNs, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastTsNs int64, err error) {
 	for {
 		n, err := r.Read(sizeBuf)
 		if err != nil {
@@ -207,7 +207,7 @@ func ReadEachLogEntry(r io.Reader, sizeBuf []byte, startTsNs, stopTsNs int64, ea
 			return lastTsNs, err
 		}
 		// println("each log: ", logEntry.TsNs)
-		if err := eachLogEntryFn(logEntry); err != nil {
+		if _, err := eachLogEntryFn(logEntry); err != nil {
 			return lastTsNs, err
 		} else {
 			lastTsNs = logEntry.TsNs

weed/filer/filerstore_wrapper.go

@@ -32,7 +32,7 @@ type VirtualFilerStore interface {
 
 type FilerStoreWrapper struct {
 	defaultStore   FilerStore
-	pathToStore    ptrie.Trie
+	pathToStore    ptrie.Trie[string]
 	storeIdToStore map[string]FilerStore
 }
 
@@ -42,7 +42,7 @@ func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper {
 	}
 	return &FilerStoreWrapper{
 		defaultStore:   store,
-		pathToStore:    ptrie.New(),
+		pathToStore:    ptrie.New[string](),
 		storeIdToStore: make(map[string]FilerStore),
 	}
 }
@@ -89,8 +89,8 @@ func (fsw *FilerStoreWrapper) getActualStore(path util.FullPath) (store FilerSto
 		return
 	}
 	var storeId string
-	fsw.pathToStore.MatchPrefix([]byte(path), func(key []byte, value interface{}) bool {
-		storeId = value.(string)
+	fsw.pathToStore.MatchPrefix([]byte(path), func(key []byte, value string) bool {
+		storeId = value
 		return false
 	})
 	if storeId != "" {
@@ -303,7 +303,7 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u
 				}
 			}
 		}
-		if count < limit && lastFileName <= prefix {
+		if count < limit && lastFileName < prefix {
 			notPrefixed = notPrefixed[:0]
 			lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool {
 				notPrefixed = append(notPrefixed, entry)

weed/filer/meta_aggregator.go

@@ -21,13 +21,13 @@ import (
 )
 
 type MetaAggregator struct {
-	filer           *Filer
-	self            pb.ServerAddress
-	isLeader        bool
-	grpcDialOption  grpc.DialOption
-	MetaLogBuffer   *log_buffer.LogBuffer
-	peerStatues     map[pb.ServerAddress]int
-	peerStatuesLock sync.Mutex
+	filer          *Filer
+	self           pb.ServerAddress
+	isLeader       bool
+	grpcDialOption grpc.DialOption
+	MetaLogBuffer  *log_buffer.LogBuffer
+	peerChans      map[pb.ServerAddress]chan struct{}
+	peerChansLock  sync.Mutex
 	// notifying clients
 	ListenersLock sync.Mutex
 	ListenersCond *sync.Cond
@@ -40,61 +40,50 @@ func NewMetaAggregator(filer *Filer, self pb.ServerAddress, grpcDialOption grpc.
 		filer:          filer,
 		self:           self,
 		grpcDialOption: grpcDialOption,
-		peerStatues:    make(map[pb.ServerAddress]int),
+		peerChans:      make(map[pb.ServerAddress]chan struct{}),
 	}
 	t.ListenersCond = sync.NewCond(&t.ListenersLock)
-	t.MetaLogBuffer = log_buffer.NewLogBuffer("aggr", LogFlushInterval, nil, func() {
+	t.MetaLogBuffer = log_buffer.NewLogBuffer("aggr", LogFlushInterval, nil, nil, func() {
 		t.ListenersCond.Broadcast()
 	})
 	return t
 }
 
 func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startFrom time.Time) {
+	ma.peerChansLock.Lock()
+	defer ma.peerChansLock.Unlock()
+
 	address := pb.ServerAddress(update.Address)
 	if update.IsAdd {
-		// every filer should subscribe to a new filer
-		if ma.setActive(address, true) {
-			go ma.loopSubscribeToOneFiler(ma.filer, ma.self, address, startFrom)
+		// cancel previous subscription if any
+		if prevChan, found := ma.peerChans[address]; found {
+			close(prevChan)
 		}
+		stopChan := make(chan struct{})
+		ma.peerChans[address] = stopChan
+		go ma.loopSubscribeToOneFiler(ma.filer, ma.self, address, startFrom, stopChan)
 	} else {
-		ma.setActive(address, false)
+		if prevChan, found := ma.peerChans[address]; found {
+			close(prevChan)
+			delete(ma.peerChans, address)
+		}
 	}
 }
 
-func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) (notDuplicated bool) {
-	ma.peerStatuesLock.Lock()
-	defer ma.peerStatuesLock.Unlock()
-	if isActive {
-		if _, found := ma.peerStatues[address]; found {
-			ma.peerStatues[address] += 1
-		} else {
-			ma.peerStatues[address] = 1
-			notDuplicated = true
-		}
-	} else {
-		if _, found := ma.peerStatues[address]; found {
-			delete(ma.peerStatues, address)
-		}
-	}
-	return
-}
-func (ma *MetaAggregator) isActive(address pb.ServerAddress) (isActive bool) {
-	ma.peerStatuesLock.Lock()
-	defer ma.peerStatuesLock.Unlock()
-	var count int
-	count, isActive = ma.peerStatues[address]
-	return count > 0 && isActive
-}
-
-func (ma *MetaAggregator) loopSubscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress, startFrom time.Time) {
+func (ma *MetaAggregator) loopSubscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress, startFrom time.Time, stopChan chan struct{}) {
 	lastTsNs := startFrom.UnixNano()
 	for {
 		glog.V(0).Infof("loopSubscribeToOneFiler read %s start from %v %d", peer, time.Unix(0, lastTsNs), lastTsNs)
 		nextLastTsNs, err := ma.doSubscribeToOneFiler(f, self, peer, lastTsNs)
-		if !ma.isActive(peer) {
-			glog.V(0).Infof("stop subscribing remote %s meta change", peer)
+
+		// check stopChan to see if we should stop
+		select {
+		case <-stopChan:
+			glog.V(0).Infof("stop subscribing peer %s meta change", peer)
 			return
+		default:
 		}
+
 		if err != nil {
 			errLvl := glog.Level(0)
 			if strings.Contains(err.Error(), "duplicated local subscription detected") {
@@ -179,7 +168,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
 		}
 		dir := event.Directory
 		// println("received meta change", dir, "size", len(data))
-		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
+		ma.MetaLogBuffer.AddDataToBuffer([]byte(dir), data, event.TsNs)
 		if maybeReplicateMetadataChange != nil {
 			maybeReplicateMetadataChange(event)
 		}
@@ -199,6 +188,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
 			ClientEpoch: atomic.LoadInt32(&ma.filer.UniqueFilerEpoch),
 		})
 		if err != nil {
+			glog.V(0).Infof("SubscribeLocalMetadata %v: %v", peer, err)
 			return fmt.Errorf("subscribe: %v", err)
 		}
 
@@ -208,10 +198,12 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
 				return nil
 			}
 			if listenErr != nil {
+				glog.V(0).Infof("SubscribeLocalMetadata stream %v: %v", peer, listenErr)
 				return listenErr
 			}
 
 			if err := processEventFn(resp); err != nil {
+				glog.V(0).Infof("SubscribeLocalMetadata process %v: %v", resp, err)
 				return fmt.Errorf("process %v: %v", resp, err)
 			}
 

weed/filer/mongodb/mongodb_store.go

@@ -2,7 +2,12 @@ package mongodb
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
+	"os"
+	"time"
+
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -10,7 +15,6 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
-	"time"
 )
 
 func init() {
@@ -37,17 +41,44 @@ func (store *MongodbStore) Initialize(configuration util.Configuration, prefix s
 	store.database = configuration.GetString(prefix + "database")
 	store.collectionName = "filemeta"
 	poolSize := configuration.GetInt(prefix + "option_pool_size")
-	return store.connection(configuration.GetString(prefix+"uri"), uint64(poolSize))
+	uri := configuration.GetString(prefix + "uri")
+	ssl := configuration.GetBool(prefix + "ssl")
+	sslCAFile := configuration.GetString(prefix + "ssl_ca_file")
+	sslCertFile := configuration.GetString(prefix + "ssl_cert_file")
+	sslKeyFile := configuration.GetString(prefix + "ssl_key_file")
+	username := configuration.GetString(prefix + "username")
+	password := configuration.GetString(prefix + "password")
+	insecure_skip_verify := configuration.GetBool(prefix + "insecure_skip_verify")
+
+	return store.connection(uri, uint64(poolSize), ssl, sslCAFile, sslCertFile, sslKeyFile, username, password, insecure_skip_verify)
 }
 
-func (store *MongodbStore) connection(uri string, poolSize uint64) (err error) {
-	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
+func (store *MongodbStore) connection(uri string, poolSize uint64, ssl bool, sslCAFile, sslCertFile, sslKeyFile string, username, password string, insecure bool) (err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
 	opts := options.Client().ApplyURI(uri)
 
 	if poolSize > 0 {
 		opts.SetMaxPoolSize(poolSize)
 	}
 
+	if ssl {
+		tlsConfig, err := configureTLS(sslCAFile, sslCertFile, sslKeyFile, insecure)
+		if err != nil {
+			return err
+		}
+		opts.SetTLSConfig(tlsConfig)
+	}
+
+	if username != "" && password != "" {
+		creds := options.Credential{
+			Username: username,
+			Password: password,
+		}
+		opts.SetAuth(creds)
+	}
+
 	client, err := mongo.Connect(ctx, opts)
 	if err != nil {
 		return err
@@ -55,10 +86,36 @@ func (store *MongodbStore) connection(uri string, poolSize uint64) (err error) {
 
 	c := client.Database(store.database).Collection(store.collectionName)
 	err = store.indexUnique(c)
+
 	store.connect = client
 	return err
 }
 
+func configureTLS(caFile, certFile, keyFile string, insecure bool) (*tls.Config, error) {
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return nil, fmt.Errorf("could not load client key pair: %s", err)
+	}
+
+	caCert, err := os.ReadFile(caFile)
+	if err != nil {
+		return nil, fmt.Errorf("could not read CA certificate: %s", err)
+	}
+
+	caCertPool := x509.NewCertPool()
+	if !caCertPool.AppendCertsFromPEM(caCert) {
+		return nil, fmt.Errorf("failed to append CA certificate")
+	}
+
+	tlsConfig := &tls.Config{
+		Certificates:       []tls.Certificate{cert},
+		RootCAs:            caCertPool,
+		InsecureSkipVerify: insecure,
+	}
+
+	return tlsConfig, nil
+}
+
 func (store *MongodbStore) createIndex(c *mongo.Collection, index mongo.IndexModel, opts *options.CreateIndexesOptions) error {
 	_, err := c.Indexes().CreateOne(context.Background(), index, opts)
 	return err
@@ -93,13 +150,10 @@ func (store *MongodbStore) RollbackTransaction(ctx context.Context) error {
 }
 
 func (store *MongodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
-
 	return store.UpdateEntry(ctx, entry)
-
 }
 
 func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
-
 	dir, name := entry.FullPath.DirAndName()
 	meta, err := entry.EncodeAttributesAndChunks()
 	if err != nil {
@@ -126,7 +180,6 @@ func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry)
 }
 
 func (store *MongodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
-
 	dir, name := fullpath.DirAndName()
 	var data Model
 
@@ -154,7 +207,6 @@ func (store *MongodbStore) FindEntry(ctx context.Context, fullpath util.FullPath
 }
 
 func (store *MongodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {
-
 	dir, name := fullpath.DirAndName()
 
 	where := bson.M{"directory": dir, "name": name}
@@ -167,7 +219,6 @@ func (store *MongodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPa
 }
 
 func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
-
 	where := bson.M{"directory": fullpath}
 	_, err := store.connect.Database(store.database).Collection(store.collectionName).DeleteMany(ctx, where)
 	if err != nil {
@@ -182,7 +233,6 @@ func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dir
 }
 
 func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
-
 	var where = bson.M{"directory": string(dirPath), "name": bson.M{"$gt": startFileName}}
 	if includeStartFile {
 		where["name"] = bson.M{

weed/filer/mysql2/mysql2_store.go

@@ -82,7 +82,7 @@ func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpse
 	store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
 
 	if err = store.DB.Ping(); err != nil {
-		return fmt.Errorf("connect to %s error:%v", sqlUrl, err)
+		return fmt.Errorf("connect to %s error:%v", adaptedSqlUrl, err)
 	}
 
 	if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil && !strings.Contains(err.Error(), "table already exist") {

weed/filer/postgres/postgres_store.go

@@ -92,7 +92,7 @@ func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, us
 	store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
 
 	if err = store.DB.Ping(); err != nil {
-		return fmt.Errorf("connect to %s error:%v", sqlUrl, err)
+		return fmt.Errorf("connect to %s error:%v", adaptedSqlUrl, err)
 	}
 
 	return nil

weed/filer/postgres2/postgres2_store.go

@@ -97,7 +97,7 @@ func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableU
 	store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
 
 	if err = store.DB.Ping(); err != nil {
-		return fmt.Errorf("connect to %s error:%v", sqlUrl, err)
+		return fmt.Errorf("connect to %s error:%v", adaptedSqlUrl, err)
 	}
 
 	if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {

weed/filer/reader_cache.go

@@ -2,6 +2,7 @@ package filer
 
 import (
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -22,16 +23,16 @@ type ReaderCache struct {
 type SingleChunkCacher struct {
 	completedTimeNew int64
 	sync.Mutex
-	parent           *ReaderCache
-	chunkFileId      string
-	data             []byte
-	err              error
-	cipherKey        []byte
-	isGzipped        bool
-	chunkSize        int
-	shouldCache      bool
-	wg               sync.WaitGroup
-	cacheStartedCh   chan struct{}
+	parent         *ReaderCache
+	chunkFileId    string
+	data           []byte
+	err            error
+	cipherKey      []byte
+	isGzipped      bool
+	chunkSize      int
+	shouldCache    bool
+	wg             sync.WaitGroup
+	cacheStartedCh chan struct{}
 }
 
 func NewReaderCache(limit int, chunkCache chunk_cache.ChunkCache, lookupFileIdFn wdclient.LookupFileIdFunctionType) *ReaderCache {
@@ -170,7 +171,7 @@ func (s *SingleChunkCacher) startCaching() {
 
 	s.data = mem.Allocate(s.chunkSize)
 
-	_, s.err = retriedFetchChunkData(s.data, urlStrings, s.cipherKey, s.isGzipped, true, 0)
+	_, s.err = util.RetriedFetchChunkData(s.data, urlStrings, s.cipherKey, s.isGzipped, true, 0)
 	if s.err != nil {
 		mem.Free(s.data)
 		s.data = nil

weed/filer/remote_storage.go

@@ -21,13 +21,13 @@ const REMOTE_STORAGE_CONF_SUFFIX = ".conf"
 const REMOTE_STORAGE_MOUNT_FILE = "mount.mapping"
 
 type FilerRemoteStorage struct {
-	rules             ptrie.Trie
+	rules             ptrie.Trie[*remote_pb.RemoteStorageLocation]
 	storageNameToConf map[string]*remote_pb.RemoteConf
 }
 
 func NewFilerRemoteStorage() (rs *FilerRemoteStorage) {
 	rs = &FilerRemoteStorage{
-		rules:             ptrie.New(),
+		rules:             ptrie.New[*remote_pb.RemoteStorageLocation](),
 		storageNameToConf: make(map[string]*remote_pb.RemoteConf),
 	}
 	return rs
@@ -82,9 +82,9 @@ func (rs *FilerRemoteStorage) mapDirectoryToRemoteStorage(dir util.FullPath, loc
 }
 
 func (rs *FilerRemoteStorage) FindMountDirectory(p util.FullPath) (mountDir util.FullPath, remoteLocation *remote_pb.RemoteStorageLocation) {
-	rs.rules.MatchPrefix([]byte(p), func(key []byte, value interface{}) bool {
+	rs.rules.MatchPrefix([]byte(p), func(key []byte, value *remote_pb.RemoteStorageLocation) bool {
 		mountDir = util.FullPath(string(key[:len(key)-1]))
-		remoteLocation = value.(*remote_pb.RemoteStorageLocation)
+		remoteLocation = value
 		return true
 	})
 	return
@@ -92,8 +92,8 @@ func (rs *FilerRemoteStorage) FindMountDirectory(p util.FullPath) (mountDir util
 
 func (rs *FilerRemoteStorage) FindRemoteStorageClient(p util.FullPath) (client remote_storage.RemoteStorageClient, remoteConf *remote_pb.RemoteConf, found bool) {
 	var storageLocation *remote_pb.RemoteStorageLocation
-	rs.rules.MatchPrefix([]byte(p), func(key []byte, value interface{}) bool {
-		storageLocation = value.(*remote_pb.RemoteStorageLocation)
+	rs.rules.MatchPrefix([]byte(p), func(key []byte, value *remote_pb.RemoteStorageLocation) bool {
+		storageLocation = value
 		return true
 	})
 

weed/filer/stream.go

@@ -69,11 +69,17 @@ func NewFileReader(filerClient filer_pb.FilerClient, entry *filer_pb.Entry) io.R
 
 type DoStreamContent func(writer io.Writer) error
 
-func PrepareStreamContent(masterClient wdclient.HasLookupFileIdFunction, chunks []*filer_pb.FileChunk, offset int64, size int64) (DoStreamContent, error) {
-	return PrepareStreamContentWithThrottler(masterClient, chunks, offset, size, 0)
+func PrepareStreamContent(masterClient wdclient.HasLookupFileIdFunction, jwtFunc VolumeServerJwtFunction, chunks []*filer_pb.FileChunk, offset int64, size int64) (DoStreamContent, error) {
+	return PrepareStreamContentWithThrottler(masterClient, jwtFunc, chunks, offset, size, 0)
 }
 
-func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunction, chunks []*filer_pb.FileChunk, offset int64, size int64, downloadMaxBytesPs int64) (DoStreamContent, error) {
+type VolumeServerJwtFunction func(fileId string) string
+
+func noJwtFunc(string) string {
+	return ""
+}
+
+func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunction, jwtFunc VolumeServerJwtFunction, chunks []*filer_pb.FileChunk, offset int64, size int64, downloadMaxBytesPs int64) (DoStreamContent, error) {
 	glog.V(4).Infof("prepare to stream content for chunks: %d", len(chunks))
 	chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
 
@@ -119,7 +125,8 @@ func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunc
 			}
 			urlStrings := fileId2Url[chunkView.FileId]
 			start := time.Now()
-			err := retriedStreamFetchChunkData(writer, urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize))
+			jwt := jwtFunc(chunkView.FileId)
+			err := retriedStreamFetchChunkData(writer, urlStrings, jwt, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize))
 			offset += int64(chunkView.ViewSize)
 			remaining -= int64(chunkView.ViewSize)
 			stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds())
@@ -143,7 +150,7 @@ func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunc
 }
 
 func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
-	streamFn, err := PrepareStreamContent(masterClient, chunks, offset, size)
+	streamFn, err := PrepareStreamContent(masterClient, noJwtFunc, chunks, offset, size)
 	if err != nil {
 		return err
 	}
@@ -187,7 +194,7 @@ func ReadAll(buffer []byte, masterClient *wdclient.MasterClient, chunks []*filer
 			return err
 		}
 
-		n, err := retriedFetchChunkData(buffer[idx:idx+int(chunkView.ViewSize)], urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk)
+		n, err := util.RetriedFetchChunkData(buffer[idx:idx+int(chunkView.ViewSize)], urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk)
 		if err != nil {
 			return err
 		}

weed/filer/ydb/ydb_store.go

@@ -79,7 +79,7 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix
 	}
 	opts := []ydb.Option{
 		ydb.WithDialTimeout(time.Duration(dialTimeOut) * time.Second),
-		environ.WithEnvironCredentials(ctx),
+		environ.WithEnvironCredentials(),
 	}
 	if poolSizeLimit > 0 {
 		opts = append(opts, ydb.WithSessionPoolSizeLimit(poolSizeLimit))

weed/ftpd/ftp_server.go

@@ -29,7 +29,7 @@ type SftpServer struct {
 
 var _ = ftpserver.MainDriver(&SftpServer{})
 
-// NewServer returns a new FTP server driver
+// NewFtpServer returns a new FTP server driver
 func NewFtpServer(ftpListener net.Listener, option *FtpServerOption) (*SftpServer, error) {
 	var err error
 	server := &SftpServer{

weed/iamapi/iamapi_management_handlers.go

@@ -3,6 +3,7 @@ package iamapi
 import (
 	"crypto/sha1"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"math/rand"
 	"net/http"
@@ -13,6 +14,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
@@ -31,6 +33,7 @@ const (
 	StatementActionReadAcp  = "GetBucketAcl"
 	StatementActionList     = "List*"
 	StatementActionTagging  = "Tagging*"
+	StatementActionDelete   = "DeleteBucket*"
 )
 
 var (
@@ -56,6 +59,8 @@ func MapToStatementAction(action string) string {
 		return s3_constants.ACTION_LIST
 	case StatementActionTagging:
 		return s3_constants.ACTION_TAGGING
+	case StatementActionDelete:
+		return s3_constants.ACTION_DELETE_BUCKET
 	default:
 		return ""
 	}
@@ -77,6 +82,8 @@ func MapToIdentitiesAction(action string) string {
 		return StatementActionList
 	case s3_constants.ACTION_TAGGING:
 		return StatementActionTagging
+	case s3_constants.ACTION_DELETE_BUCKET:
+		return StatementActionDelete
 	default:
 		return ""
 	}
@@ -424,7 +431,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 	}
 	values := r.PostForm
 	s3cfg := &iam_pb.S3ApiConfiguration{}
-	if err := iama.s3ApiConfig.GetS3ApiConfiguration(s3cfg); err != nil {
+	if err := iama.s3ApiConfig.GetS3ApiConfiguration(s3cfg); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
 		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}

weed/images/cropping.go

@@ -8,7 +8,7 @@ import (
 	"image/png"
 	"io"
 
-	"github.com/disintegration/imaging"
+	"github.com/cognusion/imaging"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 )

weed/images/resizing.go

@@ -8,7 +8,7 @@ import (
 	"image/png"
 	"io"
 
-	"github.com/disintegration/imaging"
+	"github.com/cognusion/imaging"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 

weed/mount/inode_to_path.go

@@ -114,9 +114,9 @@ func (i *InodeToPath) AllocateInode(path util.FullPath, unixTime int64) uint64 {
 	return inode
 }
 
-func (i *InodeToPath) GetInode(path util.FullPath) uint64 {
+func (i *InodeToPath) GetInode(path util.FullPath) (uint64, bool) {
 	if path == "/" {
-		return 1
+		return 1, true
 	}
 	i.Lock()
 	defer i.Unlock()
@@ -125,7 +125,7 @@ func (i *InodeToPath) GetInode(path util.FullPath) uint64 {
 		// glog.Fatalf("GetInode unknown inode for %s", path)
 		// this could be the parent for mount point
 	}
-	return inode
+	return inode, found
 }
 
 func (i *InodeToPath) GetPath(inode uint64) (util.FullPath, fuse.Status) {
@@ -146,8 +146,8 @@ func (i *InodeToPath) HasPath(path util.FullPath) bool {
 }
 
 func (i *InodeToPath) MarkChildrenCached(fullpath util.FullPath) {
-	i.RLock()
-	defer i.RUnlock()
+	i.Lock()
+	defer i.Unlock()
 	inode, found := i.path2inode[fullpath]
 	if !found {
 		// https://github.com/seaweedfs/seaweedfs/issues/4968

weed/mount/page_writer/page_chunk_mem.go

@@ -63,11 +63,11 @@ func (mc *MemChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64)
 		logicStart := max(off, memChunkBaseOffset+t.StartOffset)
 		logicStop := min(off+int64(len(p)), memChunkBaseOffset+t.stopOffset)
 		if logicStart < logicStop {
+			copy(p[logicStart-off:logicStop-off], mc.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset])
+			maxStop = max(maxStop, logicStop)
+
 			if t.TsNs >= tsNs {
-				copy(p[logicStart-off:logicStop-off], mc.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset])
-				maxStop = max(maxStop, logicStop)
-			} else {
-				println("read old data1", tsNs-t.TsNs, "ns")
+				println("read new data1", t.TsNs-tsNs, "ns")
 			}
 		}
 	}

weed/mount/page_writer/page_chunk_swapfile.go

@@ -129,15 +129,15 @@ func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop in
 		logicStart := max(off, chunkStartOffset+t.StartOffset)
 		logicStop := min(off+int64(len(p)), chunkStartOffset+t.stopOffset)
 		if logicStart < logicStop {
+			actualStart := logicStart - chunkStartOffset + int64(sc.actualChunkIndex)*sc.swapfile.chunkSize
+			if _, err := sc.swapfile.file.ReadAt(p[logicStart-off:logicStop-off], actualStart); err != nil {
+				glog.Errorf("failed to reading swap file %s: %v", sc.swapfile.file.Name(), err)
+				break
+			}
+			maxStop = max(maxStop, logicStop)
+
 			if t.TsNs >= tsNs {
-				actualStart := logicStart - chunkStartOffset + int64(sc.actualChunkIndex)*sc.swapfile.chunkSize
-				if _, err := sc.swapfile.file.ReadAt(p[logicStart-off:logicStop-off], actualStart); err != nil {
-					glog.Errorf("failed to reading swap file %s: %v", sc.swapfile.file.Name(), err)
-					break
-				}
-				maxStop = max(maxStop, logicStop)
-			} else {
-				println("read old data2", tsNs-t.TsNs, "ns")
+				println("read new data2", t.TsNs-tsNs, "ns")
 			}
 		}
 	}

weed/mount/weedfs.go

@@ -105,6 +105,29 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 		}, func(path util.FullPath) bool {
 			return wfs.inodeToPath.IsChildrenCached(path)
 		}, func(filePath util.FullPath, entry *filer_pb.Entry) {
+			// Find inode if it is not a deleted path
+			if inode, inode_found := wfs.inodeToPath.GetInode(filePath); inode_found {
+				// Find open file handle
+				if fh, fh_found := wfs.fhmap.FindFileHandle(inode); fh_found {
+					fhActiveLock := fh.wfs.fhLockTable.AcquireLock("invalidateFunc", fh.fh, util.ExclusiveLock)
+					defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
+
+					fh.entryLock.Lock()
+					defer fh.entryLock.Unlock()
+
+					// Recreate dirty pages
+					fh.dirtyPages.Destroy()
+					fh.dirtyPages = newPageWriter(fh, wfs.option.ChunkSizeLimit)
+
+					// Update handle entry
+					newentry, status := wfs.maybeLoadEntry(filePath)
+					if status == fuse.OK {
+						if fh.GetEntry() != newentry {
+							fh.SetEntry(newentry)
+						}
+					}
+				}
+			}
 		})
 	grace.OnInterrupt(func() {
 		wfs.metaCache.Shutdown()

weed/mount/weedfs_attr.go

@@ -145,7 +145,6 @@ func (wfs *WFS) setRootAttr(out *fuse.AttrOut) {
 
 func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry, calculateSize bool) {
 	out.Ino = inode
-	out.Blocks = (out.Size + blockSize - 1) / blockSize
 	setBlksize(out, blockSize)
 	if entry == nil {
 		return
@@ -159,6 +158,7 @@ func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.E
 	if entry.FileMode()&os.ModeSymlink != 0 {
 		out.Size = uint64(len(entry.Attributes.SymlinkTarget))
 	}
+	out.Blocks = (out.Size + blockSize - 1) / blockSize
 	out.Mtime = uint64(entry.Attributes.Mtime)
 	out.Ctime = uint64(entry.Attributes.Mtime)
 	out.Atime = uint64(entry.Attributes.Mtime)

weed/mount/weedfs_file_sync.go

@@ -104,9 +104,6 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
 		}
 	}
 
-	fhActiveLock := fh.wfs.fhLockTable.AcquireLock("doFlush", fh.fh, util.ExclusiveLock)
-	defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
-
 	if !fh.dirtyMetadata {
 		return fuse.OK
 	}
@@ -115,6 +112,9 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
 		return fuse.Status(syscall.ENOSPC)
 	}
 
+	fhActiveLock := fh.wfs.fhLockTable.AcquireLock("doFlush", fh.fh, util.ExclusiveLock)
+	defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
+
 	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 		fh.entryLock.Lock()
 		defer fh.entryLock.Unlock()

weed/mq/broker/broker_connect.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"io"
 	"math/rand"
@@ -14,31 +12,17 @@ import (
 )
 
 // BrokerConnectToBalancer connects to the broker balancer and sends stats
-func (b *MessageQueueBroker) BrokerConnectToBalancer(self string) error {
-	// find the lock owner
-	var brokerBalancer string
-	err := b.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		resp, err := client.FindLockOwner(context.Background(), &filer_pb.FindLockOwnerRequest{
-			Name: pub_balancer.LockBrokerBalancer,
-		})
-		if err != nil {
-			return err
-		}
-		brokerBalancer = resp.Owner
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-	b.currentBalancer = pb.ServerAddress(brokerBalancer)
+func (b *MessageQueueBroker) BrokerConnectToBalancer(brokerBalancer string, stopCh chan struct{}) error {
 
-	glog.V(0).Infof("broker %s found balancer %s", self, brokerBalancer)
+	self := string(b.option.BrokerAddress())
+
+	glog.V(0).Infof("broker %s connects to balancer %s", self, brokerBalancer)
 	if brokerBalancer == "" {
 		return fmt.Errorf("no balancer found")
 	}
 
 	// connect to the lock owner
-	err = pb.WithBrokerGrpcClient(false, brokerBalancer, b.grpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
+	return pb.WithBrokerGrpcClient(true, brokerBalancer, b.grpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
 		stream, err := client.PublisherToPubBalancer(context.Background())
 		if err != nil {
 			return fmt.Errorf("connect to balancer %v: %v", brokerBalancer, err)
@@ -56,6 +40,13 @@ func (b *MessageQueueBroker) BrokerConnectToBalancer(self string) error {
 		}
 
 		for {
+			// check if the broker is stopping
+			select {
+			case <-stopCh:
+				return nil
+			default:
+			}
+
 			stats := b.localTopicManager.CollectStats(time.Second * 5)
 			err = stream.Send(&mq_pb.PublisherToPubBalancerRequest{
 				Message: &mq_pb.PublisherToPubBalancerRequest_Stats{
@@ -68,13 +59,44 @@ func (b *MessageQueueBroker) BrokerConnectToBalancer(self string) error {
 				}
 				return fmt.Errorf("send stats message: %v", err)
 			}
-			glog.V(3).Infof("sent stats: %+v", stats)
+			// glog.V(3).Infof("sent stats: %+v", stats)
 
 			time.Sleep(time.Millisecond*5000 + time.Duration(rand.Intn(1000))*time.Millisecond)
 		}
-
-		return nil
 	})
+}
 
-	return err
+func (b *MessageQueueBroker) KeepConnectedToBrokerBalancer(newBrokerBalancerCh chan string) {
+	var stopPrevRunChan chan struct{}
+	for {
+		select {
+		case newBrokerBalancer := <-newBrokerBalancerCh:
+			if stopPrevRunChan != nil {
+				close(stopPrevRunChan)
+				stopPrevRunChan = nil
+			}
+			thisRunStopChan := make(chan struct{})
+			if newBrokerBalancer != "" {
+				stopPrevRunChan = thisRunStopChan
+				go func() {
+					for {
+						err := b.BrokerConnectToBalancer(newBrokerBalancer, thisRunStopChan)
+						if err != nil {
+							glog.V(0).Infof("connect to balancer %s: %v", newBrokerBalancer, err)
+							time.Sleep(time.Second)
+						} else {
+							break
+						}
+
+						select {
+						case <-thisRunStopChan:
+							return
+						default:
+						}
+
+					}
+				}()
+			}
+		}
+	}
 }

weed/mq/broker/broker_grpc_admin.go

@@ -2,25 +2,21 @@ package broker
 
 import (
 	"context"
-	"github.com/seaweedfs/seaweedfs/weed/cluster"
-	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 )
 
 func (b *MessageQueueBroker) FindBrokerLeader(c context.Context, request *mq_pb.FindBrokerLeaderRequest) (*mq_pb.FindBrokerLeaderResponse, error) {
 	ret := &mq_pb.FindBrokerLeaderResponse{}
-	err := b.withMasterClient(false, b.MasterClient.GetMaster(), func(client master_pb.SeaweedClient) error {
-		resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
-			ClientType: cluster.BrokerType,
-			FilerGroup: request.FilerGroup,
+	err := b.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		resp, err := client.FindLockOwner(context.Background(), &filer_pb.FindLockOwnerRequest{
+			Name: pub_balancer.LockBrokerBalancer,
 		})
 		if err != nil {
 			return err
 		}
-		if len(resp.ClusterNodes) == 0 {
-			return nil
-		}
-		ret.Broker = resp.ClusterNodes[0].Address
+		ret.Broker = resp.Owner
 		return nil
 	})
 	return ret, err

weed/mq/broker/broker_grpc_assign.go

@@ -0,0 +1,99 @@
+package broker
+
+import (
+	"context"
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"sync"
+)
+
+// AssignTopicPartitions Runs on the assigned broker, to execute the topic partition assignment
+func (b *MessageQueueBroker) AssignTopicPartitions(c context.Context, request *mq_pb.AssignTopicPartitionsRequest) (*mq_pb.AssignTopicPartitionsResponse, error) {
+	ret := &mq_pb.AssignTopicPartitionsResponse{}
+
+	// drain existing topic partition subscriptions
+	for _, assignment := range request.BrokerPartitionAssignments {
+		t := topic.FromPbTopic(request.Topic)
+		partition := topic.FromPbPartition(assignment.Partition)
+		b.accessLock.Lock()
+		if request.IsDraining {
+			// TODO drain existing topic partition subscriptions
+			b.localTopicManager.RemoveLocalPartition(t, partition)
+		} else {
+			var localPartition *topic.LocalPartition
+			if localPartition = b.localTopicManager.GetLocalPartition(t, partition); localPartition == nil {
+				localPartition = topic.NewLocalPartition(partition, b.genLogFlushFunc(t, assignment.Partition), b.genLogOnDiskReadFunc(t, assignment.Partition))
+				b.localTopicManager.AddLocalPartition(t, localPartition)
+			}
+		}
+		b.accessLock.Unlock()
+	}
+
+	// if is leader, notify the followers to drain existing topic partition subscriptions
+	if request.IsLeader {
+		for _, brokerPartition := range request.BrokerPartitionAssignments {
+			for _, follower := range brokerPartition.FollowerBrokers {
+				err := pb.WithBrokerGrpcClient(false, follower, b.grpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
+					_, err := client.AssignTopicPartitions(context.Background(), request)
+					return err
+				})
+				if err != nil {
+					return ret, err
+				}
+			}
+		}
+	}
+
+	glog.V(0).Infof("AssignTopicPartitions: topic %s partition assignments: %v", request.Topic, request.BrokerPartitionAssignments)
+	return ret, nil
+}
+
+// called by broker leader to drain existing partitions.
+// new/updated partitions will be detected by broker from the filer
+func (b *MessageQueueBroker) assignTopicPartitionsToBrokers(ctx context.Context, t *mq_pb.Topic, assignments []*mq_pb.BrokerPartitionAssignment, isAdd bool) error {
+	// notify the brokers to create the topic partitions in parallel
+	var wg sync.WaitGroup
+	for _, bpa := range assignments {
+		wg.Add(1)
+		go func(bpa *mq_pb.BrokerPartitionAssignment) {
+			defer wg.Done()
+			if doCreateErr := b.withBrokerClient(false, pb.ServerAddress(bpa.LeaderBroker), func(client mq_pb.SeaweedMessagingClient) error {
+				_, doCreateErr := client.AssignTopicPartitions(ctx, &mq_pb.AssignTopicPartitionsRequest{
+					Topic: t,
+					BrokerPartitionAssignments: []*mq_pb.BrokerPartitionAssignment{
+						{
+							Partition: bpa.Partition,
+						},
+					},
+					IsLeader:   true,
+					IsDraining: !isAdd,
+				})
+				if doCreateErr != nil {
+					if !isAdd {
+						return fmt.Errorf("drain topic %s %v on %s: %v", t, bpa.LeaderBroker, bpa.Partition, doCreateErr)
+					} else {
+						return fmt.Errorf("create topic %s %v on %s: %v", t, bpa.LeaderBroker, bpa.Partition, doCreateErr)
+					}
+				}
+				brokerStats, found := b.Balancer.Brokers.Get(bpa.LeaderBroker)
+				if !found {
+					brokerStats = pub_balancer.NewBrokerStats()
+					if !b.Balancer.Brokers.SetIfAbsent(bpa.LeaderBroker, brokerStats) {
+						brokerStats, _ = b.Balancer.Brokers.Get(bpa.LeaderBroker)
+					}
+				}
+				brokerStats.RegisterAssignment(t, bpa.Partition, isAdd)
+				return nil
+			}); doCreateErr != nil {
+				glog.Errorf("create topic %s partition %+v on %s: %v", t, bpa.Partition, bpa.LeaderBroker, doCreateErr)
+			}
+		}(bpa)
+	}
+	wg.Wait()
+
+	return nil
+}

weed/mq/broker/broker_grpc_balance.go

@@ -2,17 +2,13 @@ package broker
 
 import (
 	"context"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
 )
 
 func (b *MessageQueueBroker) BalanceTopics(ctx context.Context, request *mq_pb.BalanceTopicsRequest) (resp *mq_pb.BalanceTopicsResponse, err error) {
-	if b.currentBalancer == "" {
-		return nil, status.Errorf(codes.Unavailable, "no balancer")
-	}
-	if !b.lockAsBalancer.IsLocked() {
-		proxyErr := b.withBrokerClient(false, b.currentBalancer, func(client mq_pb.SeaweedMessagingClient) error {
+	if !b.isLockOwner() {
+		proxyErr := b.withBrokerClient(false, pb.ServerAddress(b.lockAsBalancer.LockOwner()), func(client mq_pb.SeaweedMessagingClient) error {
 			resp, err = client.BalanceTopics(ctx, request)
 			return nil
 		})

weed/mq/broker/broker_grpc_configure.go

@@ -3,7 +3,9 @@ package broker
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
+	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
@@ -15,11 +17,8 @@ import (
 // It generates an assignments based on existing allocations,
 // and then assign the partitions to the brokers.
 func (b *MessageQueueBroker) ConfigureTopic(ctx context.Context, request *mq_pb.ConfigureTopicRequest) (resp *mq_pb.ConfigureTopicResponse, err error) {
-	if b.currentBalancer == "" {
-		return nil, status.Errorf(codes.Unavailable, "no balancer")
-	}
-	if !b.lockAsBalancer.IsLocked() {
-		proxyErr := b.withBrokerClient(false, b.currentBalancer, func(client mq_pb.SeaweedMessagingClient) error {
+	if !b.isLockOwner() {
+		proxyErr := b.withBrokerClient(false, pb.ServerAddress(b.lockAsBalancer.LockOwner()), func(client mq_pb.SeaweedMessagingClient) error {
 			resp, err = client.ConfigureTopic(ctx, request)
 			return nil
 		})
@@ -29,79 +28,52 @@ func (b *MessageQueueBroker) ConfigureTopic(ctx context.Context, request *mq_pb.
 		return resp, err
 	}
 
-	ret := &mq_pb.ConfigureTopicResponse{}
-	ret.BrokerPartitionAssignments, err = b.Balancer.LookupOrAllocateTopicPartitions(request.Topic, true, request.PartitionCount)
-
-	for _, bpa := range ret.BrokerPartitionAssignments {
-		// fmt.Printf("create topic %s on %s\n", request.Topic, bpa.LeaderBroker)
-		if doCreateErr := b.withBrokerClient(false, pb.ServerAddress(bpa.LeaderBroker), func(client mq_pb.SeaweedMessagingClient) error {
-			_, doCreateErr := client.AssignTopicPartitions(ctx, &mq_pb.AssignTopicPartitionsRequest{
-				Topic: request.Topic,
-				BrokerPartitionAssignments: []*mq_pb.BrokerPartitionAssignment{
-					{
-						Partition: bpa.Partition,
-					},
-				},
-				IsLeader:   true,
-				IsDraining: false,
-			})
-			if doCreateErr != nil {
-				return fmt.Errorf("do create topic %s on %s: %v", request.Topic, bpa.LeaderBroker, doCreateErr)
-			}
-			brokerStats, found := b.Balancer.Brokers.Get(bpa.LeaderBroker)
-			if !found {
-				brokerStats = pub_balancer.NewBrokerStats()
-				if !b.Balancer.Brokers.SetIfAbsent(bpa.LeaderBroker, brokerStats) {
-					brokerStats, _ = b.Balancer.Brokers.Get(bpa.LeaderBroker)
-				}
-			}
-			brokerStats.RegisterAssignment(request.Topic, bpa.Partition)
-			return nil
-		}); doCreateErr != nil {
-			return nil, doCreateErr
+	// validate the schema
+	if request.RecordType != nil {
+		if _, err = schema.NewSchema(request.RecordType); err != nil {
+			return nil, status.Errorf(codes.InvalidArgument, "invalid record type %+v: %v", request.RecordType, err)
 		}
 	}
 
-	// TODO revert if some error happens in the middle of the assignments
+	t := topic.FromPbTopic(request.Topic)
+	var readErr, assignErr error
+	resp, readErr = b.readTopicConfFromFiler(t)
+	if readErr != nil {
+		glog.V(0).Infof("read topic %s conf: %v", request.Topic, readErr)
+	}
 
-	return ret, err
-}
-
-// AssignTopicPartitions Runs on the assigned broker, to execute the topic partition assignment
-func (b *MessageQueueBroker) AssignTopicPartitions(c context.Context, request *mq_pb.AssignTopicPartitionsRequest) (*mq_pb.AssignTopicPartitionsResponse, error) {
-	ret := &mq_pb.AssignTopicPartitionsResponse{}
-	self := pb.ServerAddress(fmt.Sprintf("%s:%d", b.option.Ip, b.option.Port))
+	if resp != nil {
+		assignErr = b.ensureTopicActiveAssignments(t, resp)
+		// no need to assign directly.
+		// The added or updated assignees will read from filer directly.
+		// The gone assignees will die by themselves.
+	}
 
-	// drain existing topic partition subscriptions
-	for _, brokerPartition := range request.BrokerPartitionAssignments {
-		localPartition := topic.FromPbBrokerPartitionAssignment(self, brokerPartition)
-		if request.IsDraining {
-			// TODO drain existing topic partition subscriptions
+	if readErr == nil && assignErr == nil && len(resp.BrokerPartitionAssignments) == int(request.PartitionCount) {
+		glog.V(0).Infof("existing topic partitions %d: %+v", len(resp.BrokerPartitionAssignments), resp.BrokerPartitionAssignments)
+		return
+	}
 
-			b.localTopicManager.RemoveTopicPartition(
-				topic.FromPbTopic(request.Topic),
-				localPartition.Partition)
-		} else {
-			b.localTopicManager.AddTopicPartition(
-				topic.FromPbTopic(request.Topic),
-				localPartition)
+	if resp != nil && len(resp.BrokerPartitionAssignments) > 0 {
+		if cancelErr := b.assignTopicPartitionsToBrokers(ctx, request.Topic, resp.BrokerPartitionAssignments, false); cancelErr != nil {
+			glog.V(1).Infof("cancel old topic %s partitions assignments %v : %v", request.Topic, resp.BrokerPartitionAssignments, cancelErr)
 		}
 	}
+	resp = &mq_pb.ConfigureTopicResponse{}
+	if b.Balancer.Brokers.IsEmpty() {
+		return nil, status.Errorf(codes.Unavailable, pub_balancer.ErrNoBroker.Error())
+	}
+	resp.BrokerPartitionAssignments = pub_balancer.AllocateTopicPartitions(b.Balancer.Brokers, request.PartitionCount)
+	resp.RecordType = request.RecordType
 
-	// if is leader, notify the followers to drain existing topic partition subscriptions
-	if request.IsLeader {
-		for _, brokerPartition := range request.BrokerPartitionAssignments {
-			for _, follower := range brokerPartition.FollowerBrokers {
-				err := pb.WithBrokerGrpcClient(false, follower, b.grpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
-					_, err := client.AssignTopicPartitions(context.Background(), request)
-					return err
-				})
-				if err != nil {
-					return ret, err
-				}
-			}
-		}
+	// save the topic configuration on filer
+	if err := b.saveTopicConfToFiler(request.Topic, resp); err != nil {
+		return nil, fmt.Errorf("configure topic: %v", err)
 	}
 
-	return ret, nil
+	b.Balancer.OnPartitionChange(request.Topic, resp.BrokerPartitionAssignments)
+
+	glog.V(0).Infof("ConfigureTopic: topic %s partition assignments: %v", request.Topic, resp.BrokerPartitionAssignments)
+
+	return resp, err
 }

weed/mq/broker/broker_grpc_lookup.go

@@ -3,28 +3,16 @@ package broker
 import (
 	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
 )
 
-// FindTopicBrokers returns the brokers that are serving the topic
-//
-//  1. lock the topic
-//
-//  2. find the topic partitions on the filer
-//     2.1 if the topic is not found, return error
-//     2.1.1 if the request is_for_subscribe, return error not found
-//     2.1.2 if the request is_for_publish, create the topic
-//     2.2 if the topic is found, return the brokers
-//
-//  3. unlock the topic
+// LookupTopicBrokers returns the brokers that are serving the topic
 func (b *MessageQueueBroker) LookupTopicBrokers(ctx context.Context, request *mq_pb.LookupTopicBrokersRequest) (resp *mq_pb.LookupTopicBrokersResponse, err error) {
-	if b.currentBalancer == "" {
-		return nil, status.Errorf(codes.Unavailable, "no balancer")
-	}
-	if !b.lockAsBalancer.IsLocked() {
-		proxyErr := b.withBrokerClient(false, b.currentBalancer, func(client mq_pb.SeaweedMessagingClient) error {
+	if !b.isLockOwner() {
+		proxyErr := b.withBrokerClient(false, pb.ServerAddress(b.lockAsBalancer.LockOwner()), func(client mq_pb.SeaweedMessagingClient) error {
 			resp, err = client.LookupTopicBrokers(ctx, request)
 			return nil
 		})
@@ -34,18 +22,23 @@ func (b *MessageQueueBroker) LookupTopicBrokers(ctx context.Context, request *mq
 		return resp, err
 	}
 
+	t := topic.FromPbTopic(request.Topic)
 	ret := &mq_pb.LookupTopicBrokersResponse{}
+	conf := &mq_pb.ConfigureTopicResponse{}
 	ret.Topic = request.Topic
-	ret.BrokerPartitionAssignments, err = b.Balancer.LookupOrAllocateTopicPartitions(ret.Topic, request.IsForPublish, 6)
+	if conf, err = b.readTopicConfFromFiler(t); err != nil {
+		glog.V(0).Infof("lookup topic %s conf: %v", request.Topic, err)
+	} else {
+		err = b.ensureTopicActiveAssignments(t, conf)
+		ret.BrokerPartitionAssignments = conf.BrokerPartitionAssignments
+	}
+
 	return ret, err
 }
 
 func (b *MessageQueueBroker) ListTopics(ctx context.Context, request *mq_pb.ListTopicsRequest) (resp *mq_pb.ListTopicsResponse, err error) {
-	if b.currentBalancer == "" {
-		return nil, status.Errorf(codes.Unavailable, "no balancer")
-	}
-	if !b.lockAsBalancer.IsLocked() {
-		proxyErr := b.withBrokerClient(false, b.currentBalancer, func(client mq_pb.SeaweedMessagingClient) error {
+	if !b.isLockOwner() {
+		proxyErr := b.withBrokerClient(false, pb.ServerAddress(b.lockAsBalancer.LockOwner()), func(client mq_pb.SeaweedMessagingClient) error {
 			resp, err = client.ListTopics(ctx, request)
 			return nil
 		})
@@ -76,3 +69,7 @@ func (b *MessageQueueBroker) ListTopics(ctx context.Context, request *mq_pb.List
 
 	return ret, nil
 }
+
+func (b *MessageQueueBroker) isLockOwner() bool {
+	return b.lockAsBalancer.LockOwner() == b.option.BrokerAddress().String()
+}

weed/mq/broker/broker_grpc_pub.go

@@ -7,6 +7,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"google.golang.org/grpc/peer"
+	"io"
 	"math/rand"
 	"net"
 	"sync/atomic"
@@ -34,104 +35,120 @@ import (
 // Subscribers needs to listen for new partitions and connect to the brokers.
 // Each subscription may not get data. It can act as a backup.
 
-func (b *MessageQueueBroker) Publish(stream mq_pb.SeaweedMessaging_PublishServer) error {
+func (b *MessageQueueBroker) PublishMessage(stream mq_pb.SeaweedMessaging_PublishMessageServer) error {
 	// 1. write to the volume server
 	// 2. find the topic metadata owning filer
 	// 3. write to the filer
 
-	var localTopicPartition *topic.LocalPartition
 	req, err := stream.Recv()
 	if err != nil {
 		return err
 	}
-	response := &mq_pb.PublishResponse{}
+	response := &mq_pb.PublishMessageResponse{}
 	// TODO check whether current broker should be the leader for the topic partition
-	ackInterval := 1
 	initMessage := req.GetInit()
-	if initMessage != nil {
-		t, p := topic.FromPbTopic(initMessage.Topic), topic.FromPbPartition(initMessage.Partition)
-		localTopicPartition = b.localTopicManager.GetTopicPartition(t, p)
-		if localTopicPartition == nil {
-			response.Error = fmt.Sprintf("topic %v partition %v not setup", initMessage.Topic, initMessage.Partition)
-			glog.Errorf("topic %v partition %v not setup", initMessage.Topic, initMessage.Partition)
-			return stream.Send(response)
-		}
-		ackInterval = int(initMessage.AckInterval)
-		stream.Send(response)
-	} else {
+	if initMessage == nil {
 		response.Error = fmt.Sprintf("missing init message")
 		glog.Errorf("missing init message")
 		return stream.Send(response)
 	}
 
+	// get or generate a local partition
+	t, p := topic.FromPbTopic(initMessage.Topic), topic.FromPbPartition(initMessage.Partition)
+	localTopicPartition, getOrGenErr := b.GetOrGenerateLocalPartition(t, p)
+	if getOrGenErr != nil {
+		response.Error = fmt.Sprintf("topic %v not found: %v", t, getOrGenErr)
+		glog.Errorf("topic %v not found: %v", t, getOrGenErr)
+		return stream.Send(response)
+	}
+
+	// connect to follower brokers
+	if followerErr := localTopicPartition.MaybeConnectToFollowers(initMessage, b.grpcDialOption); followerErr != nil {
+		response.Error = followerErr.Error()
+		glog.Errorf("MaybeConnectToFollowers: %v", followerErr)
+		return stream.Send(response)
+	}
+
+	var receivedSequence, acknowledgedSequence int64
+	var isClosed bool
+
+	// start sending ack to publisher
+	ackInterval := int64(1)
+	if initMessage.AckInterval > 0 {
+		ackInterval = int64(initMessage.AckInterval)
+	}
+	go func() {
+		defer func() {
+			// println("stop sending ack to publisher", initMessage.PublisherName)
+		}()
+
+		lastAckTime := time.Now()
+		for !isClosed {
+			receivedSequence = atomic.LoadInt64(&localTopicPartition.AckTsNs)
+			if acknowledgedSequence < receivedSequence && (receivedSequence-acknowledgedSequence >= ackInterval || time.Since(lastAckTime) > 1*time.Second) {
+				acknowledgedSequence = receivedSequence
+				response := &mq_pb.PublishMessageResponse{
+					AckSequence: acknowledgedSequence,
+				}
+				if err := stream.Send(response); err != nil {
+					glog.Errorf("Error sending response %v: %v", response, err)
+				}
+				// println("sent ack", acknowledgedSequence, "=>", initMessage.PublisherName)
+				lastAckTime = time.Now()
+			} else {
+				time.Sleep(1 * time.Second)
+			}
+		}
+	}()
+
+	// process each published messages
 	clientName := fmt.Sprintf("%v-%4d/%s/%v", findClientAddress(stream.Context()), rand.Intn(10000), initMessage.Topic, initMessage.Partition)
 	localTopicPartition.Publishers.AddPublisher(clientName, topic.NewLocalPublisher())
 
-	ackCounter := 0
-	var ackSequence int64
-	var isStopping int32
-	respChan := make(chan *mq_pb.PublishResponse, 128)
 	defer func() {
-		atomic.StoreInt32(&isStopping, 1)
-		close(respChan)
+		// remove the publisher
 		localTopicPartition.Publishers.RemovePublisher(clientName)
-	}()
-	go func() {
-		ticker := time.NewTicker(1 * time.Second)
-		for {
-			select {
-			case resp := <-respChan:
-				if resp != nil {
-					if err := stream.Send(resp); err != nil {
-						glog.Errorf("Error sending response %v: %v", resp, err)
-					}
-				} else {
-					return
-				}
-			case <-ticker.C:
-				if atomic.LoadInt32(&isStopping) == 0 {
-					response := &mq_pb.PublishResponse{
-						AckSequence: ackSequence,
-					}
-					respChan <- response
-				} else {
-					return
-				}
-			case <-localTopicPartition.StopPublishersCh:
-				respChan <- &mq_pb.PublishResponse{
-					AckSequence: ackSequence,
-					ShouldClose: true,
-				}
-			}
+		if localTopicPartition.MaybeShutdownLocalPartition() {
+			b.localTopicManager.RemoveLocalPartition(t, p)
+			glog.V(0).Infof("Removed local topic %v partition %v", initMessage.Topic, initMessage.Partition)
 		}
 	}()
 
+	// send a hello message
+	stream.Send(&mq_pb.PublishMessageResponse{})
+
+	defer func() {
+		isClosed = true
+	}()
+
 	// process each published messages
 	for {
 		// receive a message
 		req, err := stream.Recv()
 		if err != nil {
-			return err
+			if err == io.EOF {
+				break
+			}
+			glog.V(0).Infof("topic %v partition %v publish stream from %s error: %v", initMessage.Topic, initMessage.Partition, initMessage.PublisherName, err)
+			break
 		}
 
 		// Process the received message
-		if dataMessage := req.GetData(); dataMessage != nil {
-			localTopicPartition.Publish(dataMessage)
+		dataMessage := req.GetData()
+		if dataMessage == nil {
+			continue
 		}
 
-		ackCounter++
-		ackSequence++
-		if ackCounter >= ackInterval {
-			ackCounter = 0
-			// send back the ack
-			response := &mq_pb.PublishResponse{
-				AckSequence: ackSequence,
-			}
-			respChan <- response
+		// The control message should still be sent to the follower
+		// to avoid timing issue when ack messages.
+
+		// send to the local partition
+		if err = localTopicPartition.Publish(dataMessage); err != nil {
+			return fmt.Errorf("topic %v partition %v publish error: %v", initMessage.Topic, initMessage.Partition, err)
 		}
 	}
 
-	glog.V(0).Infof("topic %v partition %v publish stream closed.", initMessage.Topic, initMessage.Partition)
+	glog.V(0).Infof("topic %v partition %v publish stream from %s closed.", initMessage.Topic, initMessage.Partition, initMessage.PublisherName)
 
 	return nil
 }

weed/mq/broker/broker_grpc_pub_balancer.go

@@ -1,7 +1,6 @@
 package broker
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"google.golang.org/grpc/codes"
@@ -10,7 +9,7 @@ import (
 
 // PublisherToPubBalancer receives connections from brokers and collects stats
 func (b *MessageQueueBroker) PublisherToPubBalancer(stream mq_pb.SeaweedMessaging_PublisherToPubBalancerServer) error {
-	if !b.lockAsBalancer.IsLocked() {
+	if !b.isLockOwner() {
 		return status.Errorf(codes.Unavailable, "not current broker balancer")
 	}
 	req, err := stream.Recv()
@@ -22,12 +21,12 @@ func (b *MessageQueueBroker) PublisherToPubBalancer(stream mq_pb.SeaweedMessagin
 	initMessage := req.GetInit()
 	var brokerStats *pub_balancer.BrokerStats
 	if initMessage != nil {
-		brokerStats = b.Balancer.OnBrokerConnected(initMessage.Broker)
+		brokerStats = b.Balancer.AddBroker(initMessage.Broker)
 	} else {
 		return status.Errorf(codes.InvalidArgument, "balancer init message is empty")
 	}
 	defer func() {
-		b.Balancer.OnBrokerDisconnected(initMessage.Broker, brokerStats)
+		b.Balancer.RemoveBroker(initMessage.Broker, brokerStats)
 	}()
 
 	// process stats message
@@ -36,13 +35,12 @@ func (b *MessageQueueBroker) PublisherToPubBalancer(stream mq_pb.SeaweedMessagin
 		if err != nil {
 			return err
 		}
-		if !b.lockAsBalancer.IsLocked() {
+		if !b.isLockOwner() {
 			return status.Errorf(codes.Unavailable, "not current broker balancer")
 		}
 		if receivedStats := req.GetStats(); receivedStats != nil {
 			b.Balancer.OnBrokerStatsUpdated(initMessage.Broker, brokerStats, receivedStats)
-			glog.V(4).Infof("broker %s stats: %+v", initMessage.Broker, brokerStats)
-			glog.V(4).Infof("received stats: %+v", receivedStats)
+			// glog.V(4).Infof("received from %v: %+v", initMessage.Broker, receivedStats)
 		}
 	}
 

weed/mq/broker/broker_grpc_pub_follow.go

@@ -0,0 +1,150 @@
+package broker
+
+import (
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util/buffered_queue"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+	"io"
+	"time"
+)
+
+type memBuffer struct {
+	buf       []byte
+	startTime time.Time
+	stopTime  time.Time
+}
+
+func (b *MessageQueueBroker) PublishFollowMe(stream mq_pb.SeaweedMessaging_PublishFollowMeServer) (err error) {
+	var req *mq_pb.PublishFollowMeRequest
+	req, err = stream.Recv()
+	if err != nil {
+		return err
+	}
+	initMessage := req.GetInit()
+	if initMessage == nil {
+		return fmt.Errorf("missing init message")
+	}
+
+	// create an in-memory queue of buffered messages
+	inMemoryBuffers := buffered_queue.NewBufferedQueue[memBuffer](4)
+	logBuffer := b.buildFollowerLogBuffer(inMemoryBuffers)
+
+	lastFlushTsNs := time.Now().UnixNano()
+
+	// follow each published messages
+	for {
+		// receive a message
+		req, err = stream.Recv()
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+				break
+			}
+			glog.V(0).Infof("topic %v partition %v publish stream error: %v", initMessage.Topic, initMessage.Partition, err)
+			break
+		}
+
+		// Process the received message
+		if dataMessage := req.GetData(); dataMessage != nil {
+
+			// TODO: change this to DataMessage
+			// log the message
+			logBuffer.AddToBuffer(dataMessage)
+
+			// send back the ack
+			if err := stream.Send(&mq_pb.PublishFollowMeResponse{
+				AckTsNs: dataMessage.TsNs,
+			}); err != nil {
+				glog.Errorf("Error sending response %v: %v", dataMessage, err)
+			}
+			// println("ack", string(dataMessage.Key), dataMessage.TsNs)
+		} else if closeMessage := req.GetClose(); closeMessage != nil {
+			glog.V(0).Infof("topic %v partition %v publish stream closed: %v", initMessage.Topic, initMessage.Partition, closeMessage)
+			break
+		} else if flushMessage := req.GetFlush(); flushMessage != nil {
+			glog.V(0).Infof("topic %v partition %v publish stream flushed: %v", initMessage.Topic, initMessage.Partition, flushMessage)
+
+			lastFlushTsNs = flushMessage.TsNs
+
+			// drop already flushed messages
+			for mem, found := inMemoryBuffers.PeekHead(); found; mem, found = inMemoryBuffers.PeekHead() {
+				if mem.stopTime.UnixNano() <= flushMessage.TsNs {
+					inMemoryBuffers.Dequeue()
+					// println("dropping flushed messages: ", mem.startTime.UnixNano(), mem.stopTime.UnixNano(), len(mem.buf))
+				} else {
+					break
+				}
+			}
+
+		} else {
+			glog.Errorf("unknown message: %v", req)
+		}
+	}
+
+	t, p := topic.FromPbTopic(initMessage.Topic), topic.FromPbPartition(initMessage.Partition)
+
+	logBuffer.ShutdownLogBuffer()
+	// wait until all messages are sent to inMemoryBuffers
+	for !logBuffer.IsAllFlushed() {
+		time.Sleep(113 * time.Millisecond)
+	}
+
+	topicDir := fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
+	partitionGeneration := time.Unix(0, p.UnixTimeNs).UTC().Format(topic.TIME_FORMAT)
+	partitionDir := fmt.Sprintf("%s/%s/%04d-%04d", topicDir, partitionGeneration, p.RangeStart, p.RangeStop)
+
+	// flush the remaining messages
+	inMemoryBuffers.CloseInput()
+	for mem, found := inMemoryBuffers.Dequeue(); found; mem, found = inMemoryBuffers.Dequeue() {
+		if len(mem.buf) == 0 {
+			continue
+		}
+
+		startTime, stopTime := mem.startTime.UTC(), mem.stopTime.UTC()
+
+		if stopTime.UnixNano() <= lastFlushTsNs {
+			glog.V(0).Infof("dropping remaining data at %v %v", t, p)
+			continue
+		}
+
+		// TODO trim data earlier than lastFlushTsNs
+
+		targetFile := fmt.Sprintf("%s/%s", partitionDir, startTime.Format(topic.TIME_FORMAT))
+
+		for {
+			if err := b.appendToFile(targetFile, mem.buf); err != nil {
+				glog.V(0).Infof("metadata log write failed %s: %v", targetFile, err)
+				time.Sleep(737 * time.Millisecond)
+			} else {
+				break
+			}
+		}
+
+		glog.V(0).Infof("flushed remaining data at %v to %s size %d", mem.stopTime.UnixNano(), targetFile, len(mem.buf))
+	}
+
+	glog.V(0).Infof("shut down follower for %v %v", t, p)
+
+	return err
+}
+
+func (b *MessageQueueBroker) buildFollowerLogBuffer(inMemoryBuffers *buffered_queue.BufferedQueue[memBuffer]) *log_buffer.LogBuffer {
+	lb := log_buffer.NewLogBuffer("follower",
+		2*time.Minute, func(logBuffer *log_buffer.LogBuffer, startTime, stopTime time.Time, buf []byte) {
+			if len(buf) == 0 {
+				return
+			}
+			inMemoryBuffers.Enqueue(memBuffer{
+				buf:       buf,
+				startTime: startTime,
+				stopTime:  stopTime,
+			})
+			glog.V(0).Infof("queue up %d~%d size %d", startTime.UnixNano(), stopTime.UnixNano(), len(buf))
+		}, nil, func() {
+		})
+	return lb
+}