3.67

fix https://github.com/seaweedfs/seaweedfs/issues/5567

.github/workflows/binaries_dev.yml

@@ -38,13 +38,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -60,7 +60,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -87,13 +87,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -109,7 +109,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release0.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release1.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release2.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release3.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release4.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -45,7 +45,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@b173bce0484dc9f34c585181e5db16bd8756a21e # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/codeql.yml

@@ -18,7 +18,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL

.github/workflows/container_dev.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -36,27 +36,27 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_latest.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -37,27 +37,27 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release1.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -37,17 +37,17 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release2.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -38,17 +38,17 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release3.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -38,17 +38,17 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release4.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -37,17 +37,17 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release5.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
@@ -37,17 +37,17 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/depsreview.yml

@@ -9,6 +9,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@9129d7d40b8c12c1ed0f60400d00c92d437adcce
+        uses: actions/dependency-review-action@0659a74c94536054bfa5aeb92241f70d680cc78e

.github/workflows/e2e.yml

@@ -30,7 +30,7 @@ jobs:
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
 
     - name: Install dependencies
       run: |

.github/workflows/go.yml

@@ -27,7 +27,7 @@ jobs:
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
+      uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
 
     - name: Get dependencies
       run: |

.github/workflows/helm_chart_release.yml

@@ -12,7 +12,7 @@ jobs:
   release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: Publish Helm charts
         uses: stefanprodan/helm-gh-pages@master
         with:          

.github/workflows/helm_ci.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
         with:
           fetch-depth: 0
 
@@ -45,7 +45,7 @@ jobs:
         run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts
 
       - name: Create kind cluster
-        uses: helm/kind-action@v1.9.0
+        uses: helm/kind-action@v1.10.0
 
       - name: Run chart-testing (install)
         run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts

.github/workflows/s3tests.yml

@@ -0,0 +1,200 @@
+name: "Ceph S3 tests"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+concurrency:
+  group: ${{ github.head_ref }}/s3tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    working-directory: docker
+
+jobs:
+  s3tests:
+    name: Ceph S3 tests
+    runs-on: ubuntu-22.04
+    container:
+      image: docker.io/kmlebedev/ceph-s3-tests:0.0.2
+    timeout-minutes: 30
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Run Ceph S3 tests
+        timeout-minutes: 15
+        env:
+          S3TEST_CONF: /__w/seaweedfs/seaweedfs/docker/compose/s3tests.conf
+        shell: bash
+        run: |
+          cd /__w/seaweedfs/seaweedfs/weed
+          go install -buildvcs=false
+          set -x
+          nohup weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
+            -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
+            -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
+            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
+          sleep 10          
+          cd /s3-tests
+          tox -- \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_many \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_many \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_dot \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_one \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_one \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_zero \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_not_in_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_after_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_after_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_long_name \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \
+          s3tests_boto3/functional/test_s3.py::test_bucket_delete_notexist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_create_delete \
+          s3tests_boto3/functional/test_s3.py::test_object_read_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_multi_object_delete \
+          s3tests_boto3/functional/test_s3.py::test_multi_objectv2_delete \
+          s3tests_boto3/functional/test_s3.py::test_object_head_zero_bytes \
+          s3tests_boto3/functional/test_s3.py::test_object_write_check_etag \
+          s3tests_boto3/functional/test_s3.py::test_object_write_cache_control \
+          s3tests_boto3/functional/test_s3.py::test_object_write_expires \
+          s3tests_boto3/functional/test_s3.py::test_object_write_read_update_read_delete \
+          s3tests_boto3/functional/test_s3.py::test_object_metadata_replaced_on_put \
+          s3tests_boto3/functional/test_s3.py::test_object_write_file \
+          s3tests_boto3/functional/test_s3.py::test_post_object_invalid_date_format \
+          s3tests_boto3/functional/test_s3.py::test_post_object_no_key_specified \
+          s3tests_boto3/functional/test_s3.py::test_post_object_missing_signature \
+          s3tests_boto3/functional/test_s3.py::test_post_object_condition_is_case_sensitive \
+          s3tests_boto3/functional/test_s3.py::test_post_object_expires_is_case_sensitive \
+          s3tests_boto3/functional/test_s3.py::test_post_object_missing_expires_condition \
+          s3tests_boto3/functional/test_s3.py::test_post_object_missing_conditions_list \
+          s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_limit_exceeded \
+          s3tests_boto3/functional/test_s3.py::test_post_object_missing_content_length_argument \
+          s3tests_boto3/functional/test_s3.py::test_post_object_invalid_content_length_argument \
+          s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_below_minimum \
+          s3tests_boto3/functional/test_s3.py::test_post_object_empty_conditions \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_good \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_good \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_failed \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_failed \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
+          s3tests_boto3/functional/test_s3.py::test_bucket_head \
+          s3tests_boto3/functional/test_s3.py::test_bucket_head_notexist \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_bucket_acl \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_acl \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_gone \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero \
+          s3tests_boto3/functional/test_s3.py::test_object_anon_put \
+          s3tests_boto3/functional/test_s3.py::test_object_put_authenticated \
+          s3tests_boto3/functional/test_s3.py::test_bucket_recreate_overwrite_acl \
+          s3tests_boto3/functional/test_s3.py::test_bucket_recreate_new_acl \
+          s3tests_boto3/functional/test_s3.py::test_buckets_create_then_list \
+          s3tests_boto3/functional/test_s3.py::test_buckets_list_ctime \
+          s3tests_boto3/functional/test_s3.py::test_list_buckets_invalid_auth \
+          s3tests_boto3/functional/test_s3.py::test_list_buckets_bad_auth \
+          s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_period \
+          s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_hyphen \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_zero_size \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_same_bucket \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_to_itself \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_diff_bucket \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_canned_acl \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_bucket_not_found \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_key_not_found \
+          s3tests_boto3/functional/test_s3.py::test_multipart_copy_small \
+          s3tests_boto3/functional/test_s3.py::test_multipart_copy_without_range \
+          s3tests_boto3/functional/test_s3.py::test_multipart_copy_special_names \
+          s3tests_boto3/functional/test_s3.py::test_multipart_copy_multiple_sizes \
+          s3tests_boto3/functional/test_s3.py::test_multipart_get_part \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_empty \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_multiple_sizes \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_contents \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_size_too_small \
+          s3tests_boto3/functional/test_s3.py::test_multipart_resend_first_finishes_last \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_resend_part \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_missing_part \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_incorrect_etag \
+          s3tests_boto3/functional/test_s3.py::test_abort_multipart_upload \
+          s3tests_boto3/functional/test_s3.py::test_list_multipart_upload \
+          s3tests_boto3/functional/test_s3.py::test_atomic_read_1mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_read_4mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_read_8mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_write_1mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_write_4mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_write_8mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_1mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_4mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_8mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_multipart_upload_write \
+          s3tests_boto3/functional/test_s3.py::test_ranged_request_response_code \
+          s3tests_boto3/functional/test_s3.py::test_ranged_big_request_response_code \
+          s3tests_boto3/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \
+          s3tests_boto3/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \
+          s3tests_boto3/functional/test_s3.py::test_copy_object_ifmatch_good \
+          s3tests_boto3/functional/test_s3.py::test_copy_object_ifnonematch_failed \
+          s3tests_boto3/functional/test_s3.py::test_lifecycle_set \
+          s3tests_boto3/functional/test_s3.py::test_lifecycle_get \
+          s3tests_boto3/functional/test_s3.py::test_lifecycle_set_filter

.gitignore

@@ -87,10 +87,15 @@ other/java/hdfs/dependency-reduced-pom.xml
 
 # binary file
 weed/weed
-weed/mq/client/cmd/weed_pub/weed_pub
+weed/mq/client/cmd/weed_pub_kv/weed_pub
 docker/weed
 
 # test generated files
 weed/*/*.jpg
 docker/weed_sub
 docker/weed_pub
+weed/mq/schema/example.parquet
+docker/weed_pub_kv
+docker/weed_pub_record
+docker/weed_sub_kv
+docker/weed_sub_record

Makefile

@@ -15,7 +15,7 @@ full_install:
 	cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone"
 
 server: install
-	weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json -metricsPort=9324
+	weed -v 0 server -s3 -filer -filer.maxMB=64 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=./docker/compose/s3.json -metricsPort=9324
 
 benchmark: install warp_install
 	pkill weed || true

docker/Dockerfile.local

@@ -1,8 +1,8 @@
 FROM alpine AS final
 LABEL author="Chris Lu"
 COPY  ./weed /usr/bin/
-COPY  ./weed_pub /usr/bin/
-COPY  ./weed_sub /usr/bin/
+COPY  ./weed_pub* /usr/bin/
+COPY  ./weed_sub* /usr/bin/
 RUN mkdir -p /etc/seaweedfs
 COPY ./filer.toml /etc/seaweedfs/filer.toml
 COPY ./entrypoint.sh /entrypoint.sh

docker/Makefile

@@ -9,8 +9,10 @@ binary:
 	export SWCOMMIT=$(shell git rev-parse --short HEAD)
 	export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
 	cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w -extldflags -static $(SWLDFLAGS)" && mv weed ../docker/
-	cd ../weed/mq/client/cmd/weed_pub && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_pub ../../../../../docker/
-	cd ../weed/mq/client/cmd/weed_sub && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_sub ../../../../../docker/
+	cd ../weed/mq/client/cmd/weed_pub_kv && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_pub_kv ../../../../../docker/
+	cd ../weed/mq/client/cmd/weed_pub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_pub_record ../../../../../docker/
+	cd ../weed/mq/client/cmd/weed_sub_kv && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_sub_kv ../../../../../docker/
+	cd ../weed/mq/client/cmd/weed_sub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_sub_record ../../../../../docker/
 
 binary_race: options = -race
 binary_race: cgo = 1

docker/compose/local-brokers-compose.yml

@@ -6,7 +6,7 @@ services:
     ports:
       - 9333:9333
       - 19333:19333
-    command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    command: "-v=0 master -volumeSizeLimitMB 100 -resumeState=false -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
     environment:
       WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
       WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
@@ -16,7 +16,7 @@ services:
     ports:
       - 9334:9334
       - 19334:19334
-    command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    command: "-v=0 master -volumeSizeLimitMB 100 -resumeState=false -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
     environment:
       WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
       WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
@@ -26,7 +26,7 @@ services:
     ports:
       - 9335:9335
       - 19335:19335
-    command: "-v=1 master -volumeSizeLimitMB 100 -resumeState=false -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    command: "-v=0 master -volumeSizeLimitMB 100 -resumeState=false -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
     environment:
       WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
       WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
@@ -85,6 +85,7 @@ services:
       - master2
       - volume1
       - volume2
+      - filer1
   broker1:
     image: chrislusf/seaweedfs:local
     ports:

docker/compose/s3tests.conf

@@ -2,7 +2,7 @@
 ## this section is just used for host, port and bucket_prefix
 
 # host set for rgw in vstart.sh
-host = s3
+host = 127.0.0.1
 
 # port set for rgw in vstart.sh
 port = 8000

docker/compose/test-etcd-filer.yml

@@ -30,6 +30,7 @@ services:
     environment:
       WEED_LEVELDB2_ENABLED: 'false'
       WEED_ETCD_ENABLED: 'true'
+      WEED_ETCD_KEY_PREFIX: 'seaweedfs.'
       WEED_ETCD_SERVERS: "http://etcd:2379"
     volumes:
       - ./s3.json:/etc/seaweedfs/s3.json

go.mod

@@ -3,22 +3,21 @@ module github.com/seaweedfs/seaweedfs
 go 1.22.0
 
 require (
-	cloud.google.com/go v0.112.0 // indirect
-	cloud.google.com/go/pubsub v1.36.2
-	cloud.google.com/go/storage v1.36.0
+	cloud.google.com/go v0.112.2 // indirect
+	cloud.google.com/go/pubsub v1.37.0
+	cloud.google.com/go/storage v1.40.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Shopify/sarama v1.38.1
-	github.com/aws/aws-sdk-go v1.50.25
+	github.com/aws/aws-sdk-go v1.51.30
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
-	github.com/cenkalti/backoff/v4 v4.2.1
+	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/disintegration/imaging v1.6.2
 	github.com/dustin/go-humanize v1.0.1
 	github.com/eapache/go-resiliency v1.3.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
@@ -28,22 +27,21 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
-	github.com/fclairamb/ftpserverlib v0.22.0
+	github.com/fclairamb/ftpserverlib v0.24.0
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/go-redsync/redsync/v4 v4.11.0
-	github.com/go-sql-driver/mysql v1.7.1
+	github.com/go-redsync/redsync/v4 v4.13.0
+	github.com/go-sql-driver/mysql v1.8.1
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.6.0
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3
+	github.com/golang/protobuf v1.5.4
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.2
 	github.com/google/uuid v1.6.0
-	github.com/google/wire v0.5.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.12.1 // indirect
+	github.com/google/wire v0.6.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.3 // indirect
 	github.com/gorilla/mux v1.8.1
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -56,12 +54,11 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
-	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.17.3 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/klauspost/reedsolomon v1.12.1
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.9
-	github.com/linxGnu/grocksdb v1.8.12
+	github.com/linxGnu/grocksdb v1.8.14
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-ieproxy v0.0.11 // indirect
@@ -96,85 +93,84 @@ require (
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
 	github.com/valyala/bytebufferpool v1.0.0
-	github.com/viant/assertly v0.5.4 // indirect
-	github.com/viant/ptrie v0.3.0
-	github.com/viant/toolbox v0.33.2 // indirect
+	github.com/viant/ptrie v1.0.1
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
-	go.etcd.io/etcd/client/v3 v3.5.12
-	go.mongodb.org/mongo-driver v1.14.0
+	go.etcd.io/etcd/client/v3 v3.5.13
+	go.mongodb.org/mongo-driver v1.15.0
 	go.opencensus.io v0.24.0 // indirect
-	gocloud.dev v0.36.0
-	gocloud.dev/pubsub/natspubsub v0.36.0
-	gocloud.dev/pubsub/rabbitpubsub v0.36.0
-	golang.org/x/crypto v0.19.0 // indirect
-	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	gocloud.dev v0.37.0
+	gocloud.dev/pubsub/natspubsub v0.37.0
+	gocloud.dev/pubsub/rabbitpubsub v0.37.0
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3
 	golang.org/x/image v0.15.0
-	golang.org/x/net v0.21.0
-	golang.org/x/oauth2 v0.17.0 // indirect
-	golang.org/x/sys v0.18.0
-	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/tools v0.16.1
+	golang.org/x/net v0.25.0
+	golang.org/x/oauth2 v0.19.0 // indirect
+	golang.org/x/sys v0.20.0
+	golang.org/x/text v0.15.0 // indirect
+	golang.org/x/tools v0.21.0
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
-	google.golang.org/api v0.167.0
-	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
-	google.golang.org/grpc v1.61.1
-	google.golang.org/protobuf v1.33.0
+	google.golang.org/api v0.177.0
+	google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect
+	google.golang.org/grpc v1.63.2
+	google.golang.org/protobuf v1.34.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
-	modernc.org/cc/v3 v3.40.0 // indirect
-	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.29.0 // indirect
+	modernc.org/libc v1.49.3 // indirect
 	modernc.org/mathutil v1.6.0
-	modernc.org/memory v1.7.2 // indirect
-	modernc.org/opt v0.1.3 // indirect
-	modernc.org/sqlite v1.28.0
+	modernc.org/memory v1.8.0 // indirect
+	modernc.org/sqlite v1.29.8
 	modernc.org/strutil v1.2.0
-	modernc.org/token v1.0.1 // indirect
+	modernc.org/token v1.1.0 // indirect
 )
 
 require (
 	github.com/Jille/raft-grpc-transport v1.5.0
-	github.com/arangodb/go-driver v1.6.1
+	github.com/arangodb/go-driver v1.6.2
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.25.2
-	github.com/aws/aws-sdk-go-v2/config v1.26.6
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.4
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0
+	github.com/aws/aws-sdk-go-v2 v1.26.1
+	github.com/aws/aws-sdk-go-v2/config v1.27.11
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.11
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
+	github.com/cognusion/imaging v1.0.1
 	github.com/fluent/fluent-logger-golang v1.9.0
-	github.com/golang-jwt/jwt/v5 v5.2.0
+	github.com/getsentry/sentry-go v0.27.0
+	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
-	github.com/hanwen/go-fuse/v2 v2.5.0
-	github.com/hashicorp/raft v1.6.0
+	github.com/hanwen/go-fuse/v2 v2.5.1
+	github.com/hashicorp/raft v1.6.1
 	github.com/hashicorp/raft-boltdb/v2 v2.3.0
 	github.com/orcaman/concurrent-map/v2 v2.0.1
-	github.com/puzpuzpuz/xsync/v2 v2.5.1
+	github.com/parquet-go/parquet-go v0.21.0
 	github.com/rabbitmq/amqp091-go v1.9.0
-	github.com/rclone/rclone v1.65.2
-	github.com/rdleal/intervalst v1.3.0
-	github.com/schollz/progressbar/v3 v3.14.1
-	github.com/shirou/gopsutil/v3 v3.24.1
+	github.com/rclone/rclone v1.66.0
+	github.com/rdleal/intervalst v1.4.0
+	github.com/schollz/progressbar/v3 v3.14.2
+	github.com/shirou/gopsutil/v3 v3.24.3
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.56.1
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.65.0
+	go.etcd.io/etcd/client/pkg/v3 v3.5.13
 	go.uber.org/atomic v1.11.0
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
 )
 
 require (
-	cloud.google.com/go/compute v1.24.0 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.1.6 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.0 // indirect
+	cloud.google.com/go/auth v0.3.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
+	cloud.google.com/go/compute/metadata v0.3.0 // indirect
+	cloud.google.com/go/iam v1.1.7 // indirect
+	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.1 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
@@ -186,26 +182,27 @@ require (
 	github.com/PuerkitoBio/goquery v1.8.1 // indirect
 	github.com/Unknwon/goconfig v1.0.0 // indirect
 	github.com/abbot/go-http-auth v0.4.0 // indirect
+	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.26.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.29.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.1 // indirect
-	github.com/aws/smithy-go v1.20.1 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
+	github.com/aws/smithy-go v1.20.2 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
-	github.com/bradenaw/juniper v0.13.1 // indirect
+	github.com/bradenaw/juniper v0.15.2 // indirect
 	github.com/buengese/sgzip v0.1.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
@@ -217,16 +214,16 @@ require (
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
-	github.com/emersion/go-message v0.17.0 // indirect
+	github.com/emersion/go-message v0.18.0 // indirect
 	github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
 	github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect
 	github.com/fatih/color v1.14.1 // indirect
-	github.com/fclairamb/go-log v0.4.1 // indirect
+	github.com/fclairamb/go-log v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/flynn/noise v1.0.0 // indirect
+	github.com/flynn/noise v1.0.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
-	github.com/go-chi/chi/v5 v5.0.10 // indirect
+	github.com/go-chi/chi/v5 v5.0.11 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
@@ -234,17 +231,16 @@ require (
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
-	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
-	github.com/hashicorp/go-hclog v1.5.0 // indirect
+	github.com/hashicorp/go-hclog v1.6.2 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/henrybear327/Proton-API-Bridge v1.0.0 // indirect
 	github.com/henrybear327/go-proton-api v1.0.0 // indirect
-	github.com/iguanesolutions/go-systemd/v5 v5.1.1 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
@@ -253,9 +249,9 @@ require (
 	github.com/jonboulle/clockwork v0.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jtolio/eventkit v0.0.0-20231019094657-5d77ebb407d9 // indirect
-	github.com/jtolio/noiseconn v0.0.0-20230621152802-afeab29449e0 // indirect
+	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
@@ -267,34 +263,35 @@ require (
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/montanaflynn/stats v0.7.0 // indirect
-	github.com/nats-io/nats.go v1.31.0 // indirect
-	github.com/nats-io/nkeys v0.4.6 // indirect
+	github.com/nats-io/nats.go v1.33.1 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/ncw/swift/v2 v2.0.2 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/onsi/gomega v1.27.10 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/oracle/oci-go-sdk/v65 v65.51.0 // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.55.1 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
 	github.com/philhofer/fwd v1.1.2 // indirect
-	github.com/pierrec/lz4/v4 v4.1.17 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
 	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
 	github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
-	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/sftp v1.13.6 // indirect
 	github.com/pkg/xattr v0.4.9 // indirect
 	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
 	github.com/relvacode/iso8601 v1.3.0 // indirect
 	github.com/rfjakob/eme v1.1.2 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/segmentio/encoding v0.3.6 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/smartystreets/goconvey v1.8.1 // indirect
@@ -303,49 +300,48 @@ require (
 	github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca // indirect
+	github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 // indirect
 	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
 	github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
 	github.com/tinylib/msgp v1.1.8 // indirect
-	github.com/tklauser/go-sysconf v0.3.12 // indirect
-	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/tklauser/go-sysconf v0.3.13 // indirect
+	github.com/tklauser/numcpus v0.7.0 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
-	github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
 	github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf // indirect
 	github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect
 	github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
-	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
 	github.com/zeebo/errs v1.3.0 // indirect
 	go.etcd.io/bbolt v1.3.8 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.12 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.12 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.48.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0 // indirect
-	go.opentelemetry.io/otel v1.23.0 // indirect
-	go.opentelemetry.io/otel/metric v1.23.0 // indirect
-	go.opentelemetry.io/otel/trace v1.23.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.13 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.26.0 // indirect
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/term v0.17.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/term v0.20.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	lukechampine.com/uint128 v1.2.0 // indirect
-	storj.io/common v0.0.0-20231027080355-b4cb1b0d728e // indirect
+	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
+	storj.io/common v0.0.0-20240111121419-ecae1362576c // indirect
 	storj.io/drpc v0.0.33 // indirect
+	storj.io/infectious v0.0.2 // indirect
 	storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c // indirect
-	storj.io/uplink v1.12.1 // indirect
+	storj.io/uplink v1.12.2 // indirect
 )
 
 // replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft

k8s/charts/seaweedfs/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.64"
-version: 3.64.0
+appVersion: "3.67"
+version: 3.67.0

k8s/charts/seaweedfs/README.md

@@ -30,11 +30,11 @@ so your deployment will be spread/HA.
 ## Prerequisites
 ### Database
 
-leveldb is the default database this only supports one filer replica.
+leveldb is the default database, this supports multiple filer replicas that will [sync automatically](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication), with some [limitations](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication#limitation).
 
-To have multiple filers a external datastore is recommened.
+When the [limitations](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication#limitation) apply, or for a large number of filer replicas, an external datastore is recommended.
 
-Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`. 
+Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
 This database should be pre-configured and initialized by running:
 ```sql
 CREATE TABLE IF NOT EXISTS `filemeta` (
@@ -96,7 +96,7 @@ filer:
     enabled: true
 ```
 
-### Enabling Authenticaion to S3
+### Enabling Authentication to S3
 
 To enable authentication for S3, you have two options:
 

k8s/charts/seaweedfs/templates/filer-ingress.yaml

@@ -11,9 +11,9 @@ kind: Ingress
 metadata:
   name: ingress-{{ template "seaweedfs.name" . }}-filer
   namespace: {{ .Release.Namespace }}
-  {{- if .Values.filer.ingress.annotations }}
+  {{- with .Values.filer.ingress.annotations }}
   annotations:
-    {{ tpl .Values.filer.ingress.annotations . | nindent 4 | trim }}
+    {{- toYaml . | nindent 4 }}
   {{- end }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}

k8s/charts/seaweedfs/templates/master-ingress.yaml

@@ -11,9 +11,9 @@ kind: Ingress
 metadata:
   name: ingress-{{ template "seaweedfs.name" . }}-master
   namespace: {{ .Release.Namespace }}
-  {{- if .Values.master.ingress.annotations }}
+  {{- with .Values.master.ingress.annotations }}
   annotations:
-    {{ tpl .Values.master.ingress.annotations . | nindent 4 | trim }}
+    {{- toYaml . | nindent 4 }}
   {{- end }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}

k8s/charts/seaweedfs/values.yaml

@@ -77,7 +77,7 @@ master:
 
   config: |-
     # Enter any extra configuration for master.toml here.
-    # It may be be a multi-line string.
+    # It may be a multi-line string.
 
   # You may use ANY storage-class, example with local-path-provisioner
   # Annotations are optional.
@@ -171,7 +171,7 @@ master:
     className: "nginx"
     # host: false for "*" hostname
     host: "master.seaweedfs.local"
-    annotations: |
+    annotations:
       nginx.ingress.kubernetes.io/auth-type: "basic"
       nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
       nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master'
@@ -293,9 +293,9 @@ volume:
 
   # same applies to "logs"
 
-  idx: ""
+  idx: {}
 
-  logs: ""
+  logs: {}
 
   # limit background compaction or copying speed in mega bytes per second
   compactionMBps: "50"
@@ -540,7 +540,7 @@ filer:
     className: "nginx"
     # host: false for "*" hostname
     host: "seaweedfs.cluster.local"
-    annotations: |
+    annotations:
       nginx.ingress.kubernetes.io/backend-protocol: GRPC
       nginx.ingress.kubernetes.io/auth-type: "basic"
       nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
@@ -746,7 +746,7 @@ s3:
     # host: false for "*" hostname
     host: "seaweedfs.cluster.local"
     # additional ingress annotations for the s3 endpoint
-    annotations: []
+    annotations: {}
     tls: []
 
 certificates:

weed/cluster/lock_manager/distributed_lock_manager.go

@@ -55,7 +55,8 @@ func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, move
 		return
 	}
 	if movedTo != dlm.Host {
-		glog.V(0).Infof("lock %s not on current %s but on %s", key, dlm.Host, movedTo)
+		servers := dlm.LockRing.GetSnapshot()
+		glog.V(0).Infof("lock %s not on current %s but on %s from %v", key, dlm.Host, movedTo, servers)
 		return
 	}
 	owner, err = dlm.lockManager.GetLockOwner(key)

weed/cluster/lock_manager/lock_ring.go

@@ -1,6 +1,7 @@
 package lock_manager
 
 import (
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"sort"
@@ -39,9 +40,11 @@ func (r *LockRing) SetTakeSnapshotCallback(onTakeSnapshot func(snapshot []pb.Ser
 // AddServer adds a server to the ring
 // if the previous snapshot passed the snapshot interval, create a new snapshot
 func (r *LockRing) AddServer(server pb.ServerAddress) {
+	glog.V(0).Infof("add server %v", server)
 	r.Lock()
 
 	if _, found := r.candidateServers[server]; found {
+		glog.V(0).Infof("add server: already exists %v", server)
 		r.Unlock()
 		return
 	}
@@ -53,6 +56,8 @@ func (r *LockRing) AddServer(server pb.ServerAddress) {
 }
 
 func (r *LockRing) RemoveServer(server pb.ServerAddress) {
+	glog.V(0).Infof("remove server %v", server)
+
 	r.Lock()
 
 	if _, found := r.candidateServers[server]; !found {
@@ -72,7 +77,13 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
 		return servers[i] < servers[j]
 	})
 
+	r.Lock()
 	r.lastUpdateTime = time.Now()
+	// init candidateServers
+	for _, server := range servers {
+		r.candidateServers[server] = struct{}{}
+	}
+	r.Unlock()
 
 	r.addOneSnapshot(servers)
 

weed/command/filer_sync.go

@@ -394,7 +394,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			glog.V(0).Infof("received %v", resp)
 		}
 
-		if isMultipartUploadDir(resp.Directory) {
+		if isMultipartUploadDir(resp.Directory + "/") {
 			return nil
 		}
 

weed/command/scaffold/filer.toml

@@ -270,6 +270,12 @@ username = ""
 password = ""
 key_prefix = "seaweedfs."
 timeout = "3s"
+# Set the CA certificate path
+tls_ca_file=""
+# Set the client certificate path
+tls_client_crt_file=""
+# Set the client private key path
+tls_client_key_file=""
 
 [mongodb]
 enabled = false

weed/command/shell.go

@@ -19,7 +19,7 @@ func init() {
 	cmdShell.Run = runShell // break init cycle
 	shellOptions.Masters = cmdShell.Flag.String("master", "", "comma-separated master servers, e.g. localhost:9333")
 	shellOptions.FilerGroup = cmdShell.Flag.String("filerGroup", "", "filerGroup for the filers")
-	shellInitialFiler = cmdShell.Flag.String("filer", "", "filer host and port, e.g. localhost:8888")
+	shellInitialFiler = cmdShell.Flag.String("filer", "", "filer host and port for initial connection, e.g. localhost:8888")
 	shellCluster = cmdShell.Flag.String("cluster", "", "cluster defined in shell.toml")
 }
 
@@ -30,32 +30,36 @@ var cmdShell = &Command{
 
 	Generate shell.toml via "weed scaffold -config=shell"
 
-  `,
+`,
 }
 
 func runShell(command *Command, args []string) bool {
 
 	util.LoadConfiguration("security", false)
 	shellOptions.GrpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
+	shellOptions.Directory = "/"
+
+	util.LoadConfiguration("shell", false)
+	viper := util.GetViper()
+	cluster := viper.GetString("cluster.default")
+	if *shellCluster != "" {
+		cluster = *shellCluster
+	}
 
 	if *shellOptions.Masters == "" {
-		util.LoadConfiguration("shell", false)
-		v := util.GetViper()
-		cluster := v.GetString("cluster.default")
-		if *shellCluster != "" {
-			cluster = *shellCluster
-		}
 		if cluster == "" {
 			*shellOptions.Masters = "localhost:9333"
 		} else {
-			*shellOptions.Masters = v.GetString("cluster." + cluster + ".master")
-			*shellInitialFiler = v.GetString("cluster." + cluster + ".filer")
-			fmt.Printf("master: %s filer: %s\n", *shellOptions.Masters, *shellInitialFiler)
+			*shellOptions.Masters = viper.GetString("cluster." + cluster + ".master")
 		}
 	}
 
-	shellOptions.FilerAddress = pb.ServerAddress(*shellInitialFiler)
-	shellOptions.Directory = "/"
+	filerAddress := *shellInitialFiler
+	if filerAddress == "" && cluster != "" {
+		filerAddress = viper.GetString("cluster." + cluster + ".filer")
+	}
+	shellOptions.FilerAddress = pb.ServerAddress(filerAddress)
+	fmt.Printf("master: %s filer: %s\n", *shellOptions.Masters, shellOptions.FilerAddress)
 
 	shell.RunShell(shellOptions)
 

weed/filer/etcd/etcd_store.go

@@ -1,9 +1,10 @@
 package etcd
 
 import (
-	"bytes"
 	"context"
+	"crypto/tls"
 	"fmt"
+	"go.etcd.io/etcd/client/pkg/v3/transport"
 	"strings"
 	"time"
 
@@ -26,49 +27,78 @@ func init() {
 type EtcdStore struct {
 	client        *clientv3.Client
 	etcdKeyPrefix string
+	timeout       time.Duration
 }
 
 func (store *EtcdStore) GetName() string {
 	return "etcd"
 }
 
-func (store *EtcdStore) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
+func (store *EtcdStore) Initialize(configuration weed_util.Configuration, prefix string) error {
+	configuration.SetDefault(prefix+"servers", "localhost:2379")
+	configuration.SetDefault(prefix+"timeout", "3s")
+
 	servers := configuration.GetString(prefix + "servers")
-	if servers == "" {
-		servers = "localhost:2379"
-	}
 
 	username := configuration.GetString(prefix + "username")
 	password := configuration.GetString(prefix + "password")
 	store.etcdKeyPrefix = configuration.GetString(prefix + "key_prefix")
 
-	timeout := configuration.GetString(prefix + "timeout")
-	if timeout == "" {
-		timeout = "3s"
+	timeoutStr := configuration.GetString(prefix + "timeout")
+	timeout, err := time.ParseDuration(timeoutStr)
+	if err != nil {
+		return fmt.Errorf("parse etcd store timeout: %v", err)
+	}
+	store.timeout = timeout
+
+	certFile := configuration.GetString(prefix + "tls_client_crt_file")
+	keyFile := configuration.GetString(prefix + "tls_client_key_file")
+	caFile := configuration.GetString(prefix + "tls_ca_file")
+
+	var tlsConfig *tls.Config
+	if caFile != "" {
+		tlsInfo := transport.TLSInfo{
+			CertFile:      certFile,
+			KeyFile:       keyFile,
+			TrustedCAFile: caFile,
+		}
+		var err error
+		tlsConfig, err = tlsInfo.ClientConfig()
+		if err != nil {
+			return fmt.Errorf("TLS client configuration error: %v", err)
+		}
 	}
 
-	return store.initialize(servers, username, password, timeout)
+	return store.initialize(servers, username, password, store.timeout, tlsConfig)
 }
 
-func (store *EtcdStore) initialize(servers string, username string, password string, timeout string) (err error) {
+func (store *EtcdStore) initialize(servers, username, password string, timeout time.Duration, tlsConfig *tls.Config) error {
 	glog.Infof("filer store etcd: %s", servers)
 
-	to, err := time.ParseDuration(timeout)
-	if err != nil {
-		return fmt.Errorf("parse timeout %s: %s", timeout, err)
-	}
-
-	store.client, err = clientv3.New(clientv3.Config{
+	client, err := clientv3.New(clientv3.Config{
 		Endpoints:   strings.Split(servers, ","),
 		Username:    username,
 		Password:    password,
-		DialTimeout: to,
+		DialTimeout: timeout,
+		TLS:         tlsConfig,
 	})
 	if err != nil {
 		return fmt.Errorf("connect to etcd %s: %s", servers, err)
 	}
 
-	return
+	ctx, cancel := context.WithTimeout(context.Background(), store.timeout)
+	defer cancel()
+
+	resp, err := client.Status(ctx, client.Endpoints()[0])
+	if err != nil {
+		client.Close()
+		return fmt.Errorf("error checking etcd connection: %s", err)
+	}
+
+	glog.V(0).Infof("сonnection to etcd has been successfully verified. etcd version: %s", resp.Version)
+	store.client = client
+
+	return nil
 }
 
 func (store *EtcdStore) BeginTransaction(ctx context.Context) (context.Context, error) {
@@ -148,26 +178,20 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_
 }
 
 func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
-	return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
-}
-
-func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
-	directoryPrefix := genDirectoryKeyPrefix(dirPath, "")
+	directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix)
 	lastFileStart := directoryPrefix
 	if startFileName != "" {
 		lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
 	}
 
 	resp, err := store.client.Get(ctx, store.etcdKeyPrefix+string(lastFileStart),
-		clientv3.WithFromKey(), clientv3.WithLimit(limit+1))
+		clientv3.WithRange(clientv3.GetPrefixRangeEnd(store.etcdKeyPrefix+string(directoryPrefix))),
+		clientv3.WithLimit(limit+1))
 	if err != nil {
 		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
 	}
 
 	for _, kv := range resp.Kvs {
-		if !bytes.HasPrefix(kv.Key, directoryPrefix) {
-			break
-		}
 		fileName := getNameFromKey(kv.Key)
 		if fileName == "" {
 			continue
@@ -196,6 +220,10 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u
 	return lastFileName, err
 }
 
+func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
+}
+
 func genKey(dirPath, fileName string) (key []byte) {
 	key = []byte(dirPath)
 	key = append(key, DIR_FILE_SEPARATOR)

weed/filer/etcd/etcd_store_test.go

@@ -10,7 +10,7 @@ func TestStore(t *testing.T) {
 	// to set up local env
 	if false {
 		store := &EtcdStore{}
-		store.initialize("localhost:2379", "", "", "3s")
+		store.initialize("localhost:2379", "", "", 3, nil)
 		store_test.TestFilerStore(t, store)
 	}
 }

weed/filer/filechunk_manifest.go

@@ -107,7 +107,7 @@ func fetchWholeChunk(bytesBuffer *bytes.Buffer, lookupFileIdFn wdclient.LookupFi
 		glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err)
 		return err
 	}
-	err = retriedStreamFetchChunkData(bytesBuffer, urlStrings, cipherKey, isGzipped, true, 0, 0)
+	err = retriedStreamFetchChunkData(bytesBuffer, urlStrings, "", cipherKey, isGzipped, true, 0, 0)
 	if err != nil {
 		return err
 	}
@@ -123,7 +123,7 @@ func fetchChunkRange(buffer []byte, lookupFileIdFn wdclient.LookupFileIdFunction
 	return util.RetriedFetchChunkData(buffer, urlStrings, cipherKey, isGzipped, false, offset)
 }
 
-func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) {
+func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, jwt string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) {
 
 	var shouldRetry bool
 	var totalWritten int
@@ -132,7 +132,7 @@ func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, cipherKe
 		for _, urlString := range urlStrings {
 			var localProcessed int
 			var writeErr error
-			shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
+			shouldRetry, err = util.ReadUrlAsStreamAuthenticated(urlString+"?readDeleted=true", jwt, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
 				if totalWritten > localProcessed {
 					toBeSkipped := totalWritten - localProcessed
 					if len(data) <= toBeSkipped {

weed/filer/filer_conf.go

@@ -27,7 +27,7 @@ const (
 )
 
 type FilerConf struct {
-	rules ptrie.Trie
+	rules ptrie.Trie[*filer_pb.FilerConf_PathConf]
 }
 
 func ReadFilerConf(filerGrpcAddress pb.ServerAddress, grpcDialOption grpc.DialOption, masterClient *wdclient.MasterClient) (*FilerConf, error) {
@@ -55,7 +55,7 @@ func ReadFilerConf(filerGrpcAddress pb.ServerAddress, grpcDialOption grpc.DialOp
 
 func NewFilerConf() (fc *FilerConf) {
 	fc = &FilerConf{
-		rules: ptrie.New(),
+		rules: ptrie.New[*filer_pb.FilerConf_PathConf](),
 	}
 	return fc
 }
@@ -120,8 +120,8 @@ func (fc *FilerConf) AddLocationConf(locConf *filer_pb.FilerConf_PathConf) (err
 }
 
 func (fc *FilerConf) DeleteLocationConf(locationPrefix string) {
-	rules := ptrie.New()
-	fc.rules.Walk(func(key []byte, value interface{}) bool {
+	rules := ptrie.New[*filer_pb.FilerConf_PathConf]()
+	fc.rules.Walk(func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
 		if string(key) == locationPrefix {
 			return true
 		}
@@ -135,9 +135,8 @@ func (fc *FilerConf) DeleteLocationConf(locationPrefix string) {
 
 func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf_PathConf) {
 	pathConf = &filer_pb.FilerConf_PathConf{}
-	fc.rules.MatchPrefix([]byte(path), func(key []byte, value interface{}) bool {
-		t := value.(*filer_pb.FilerConf_PathConf)
-		mergePathConf(pathConf, t)
+	fc.rules.MatchPrefix([]byte(path), func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
+		mergePathConf(pathConf, value)
 		return true
 	})
 	return pathConf
@@ -145,10 +144,9 @@ func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf
 
 func (fc *FilerConf) GetCollectionTtls(collection string) (ttls map[string]string) {
 	ttls = make(map[string]string)
-	fc.rules.Walk(func(key []byte, value interface{}) bool {
-		t := value.(*filer_pb.FilerConf_PathConf)
-		if t.Collection == collection {
-			ttls[t.LocationPrefix] = t.GetTtl()
+	fc.rules.Walk(func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
+		if value.Collection == collection {
+			ttls[value.LocationPrefix] = value.GetTtl()
 		}
 		return true
 	})
@@ -176,9 +174,8 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
 
 func (fc *FilerConf) ToProto() *filer_pb.FilerConf {
 	m := &filer_pb.FilerConf{}
-	fc.rules.Walk(func(key []byte, value interface{}) bool {
-		pathConf := value.(*filer_pb.FilerConf_PathConf)
-		m.Locations = append(m.Locations, pathConf)
+	fc.rules.Walk(func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
+		m.Locations = append(m.Locations, value)
 		return true
 	})
 	return m

weed/filer/filer_delete_entry.go

@@ -38,7 +38,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("delete directory %s: %v", p, err)
+			glog.V(2).Infof("delete directory %s: %v", p, err)
 			return fmt.Errorf("delete directory %s: %v", p, err)
 		}
 	}
@@ -76,7 +76,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 			}
 			if lastFileName == "" && !isRecursive && len(entries) > 0 {
 				// only for first iteration in the loop
-				glog.V(0).Infof("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name())
+				glog.V(2).Infof("deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name())
 				return fmt.Errorf("%s: %s", MsgFailDelNonEmptyFolder, entry.FullPath)
 			}
 

weed/filer/filer_notify.go

@@ -83,7 +83,7 @@ func (f *Filer) logMetaEvent(ctx context.Context, fullpath string, eventNotifica
 		return
 	}
 
-	f.LocalMetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
+	f.LocalMetaLogBuffer.AddDataToBuffer([]byte(dir), data, event.TsNs)
 
 }
 

weed/filer/filerstore_wrapper.go

@@ -32,7 +32,7 @@ type VirtualFilerStore interface {
 
 type FilerStoreWrapper struct {
 	defaultStore   FilerStore
-	pathToStore    ptrie.Trie
+	pathToStore    ptrie.Trie[string]
 	storeIdToStore map[string]FilerStore
 }
 
@@ -42,7 +42,7 @@ func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper {
 	}
 	return &FilerStoreWrapper{
 		defaultStore:   store,
-		pathToStore:    ptrie.New(),
+		pathToStore:    ptrie.New[string](),
 		storeIdToStore: make(map[string]FilerStore),
 	}
 }
@@ -89,8 +89,8 @@ func (fsw *FilerStoreWrapper) getActualStore(path util.FullPath) (store FilerSto
 		return
 	}
 	var storeId string
-	fsw.pathToStore.MatchPrefix([]byte(path), func(key []byte, value interface{}) bool {
-		storeId = value.(string)
+	fsw.pathToStore.MatchPrefix([]byte(path), func(key []byte, value string) bool {
+		storeId = value
 		return false
 	})
 	if storeId != "" {
@@ -303,7 +303,7 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u
 				}
 			}
 		}
-		if count < limit && lastFileName <= prefix {
+		if count < limit && lastFileName < prefix {
 			notPrefixed = notPrefixed[:0]
 			lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool {
 				notPrefixed = append(notPrefixed, entry)

weed/filer/meta_aggregator.go

@@ -168,7 +168,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
 		}
 		dir := event.Directory
 		// println("received meta change", dir, "size", len(data))
-		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
+		ma.MetaLogBuffer.AddDataToBuffer([]byte(dir), data, event.TsNs)
 		if maybeReplicateMetadataChange != nil {
 			maybeReplicateMetadataChange(event)
 		}

weed/filer/mysql2/mysql2_store.go

@@ -82,7 +82,7 @@ func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpse
 	store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
 
 	if err = store.DB.Ping(); err != nil {
-		return fmt.Errorf("connect to %s error:%v", sqlUrl, err)
+		return fmt.Errorf("connect to %s error:%v", adaptedSqlUrl, err)
 	}
 
 	if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil && !strings.Contains(err.Error(), "table already exist") {

weed/filer/postgres/postgres_store.go

@@ -92,7 +92,7 @@ func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, us
 	store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
 
 	if err = store.DB.Ping(); err != nil {
-		return fmt.Errorf("connect to %s error:%v", sqlUrl, err)
+		return fmt.Errorf("connect to %s error:%v", adaptedSqlUrl, err)
 	}
 
 	return nil

weed/filer/postgres2/postgres2_store.go

@@ -97,7 +97,7 @@ func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableU
 	store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
 
 	if err = store.DB.Ping(); err != nil {
-		return fmt.Errorf("connect to %s error:%v", sqlUrl, err)
+		return fmt.Errorf("connect to %s error:%v", adaptedSqlUrl, err)
 	}
 
 	if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {

weed/filer/remote_storage.go

@@ -21,13 +21,13 @@ const REMOTE_STORAGE_CONF_SUFFIX = ".conf"
 const REMOTE_STORAGE_MOUNT_FILE = "mount.mapping"
 
 type FilerRemoteStorage struct {
-	rules             ptrie.Trie
+	rules             ptrie.Trie[*remote_pb.RemoteStorageLocation]
 	storageNameToConf map[string]*remote_pb.RemoteConf
 }
 
 func NewFilerRemoteStorage() (rs *FilerRemoteStorage) {
 	rs = &FilerRemoteStorage{
-		rules:             ptrie.New(),
+		rules:             ptrie.New[*remote_pb.RemoteStorageLocation](),
 		storageNameToConf: make(map[string]*remote_pb.RemoteConf),
 	}
 	return rs
@@ -82,9 +82,9 @@ func (rs *FilerRemoteStorage) mapDirectoryToRemoteStorage(dir util.FullPath, loc
 }
 
 func (rs *FilerRemoteStorage) FindMountDirectory(p util.FullPath) (mountDir util.FullPath, remoteLocation *remote_pb.RemoteStorageLocation) {
-	rs.rules.MatchPrefix([]byte(p), func(key []byte, value interface{}) bool {
+	rs.rules.MatchPrefix([]byte(p), func(key []byte, value *remote_pb.RemoteStorageLocation) bool {
 		mountDir = util.FullPath(string(key[:len(key)-1]))
-		remoteLocation = value.(*remote_pb.RemoteStorageLocation)
+		remoteLocation = value
 		return true
 	})
 	return
@@ -92,8 +92,8 @@ func (rs *FilerRemoteStorage) FindMountDirectory(p util.FullPath) (mountDir util
 
 func (rs *FilerRemoteStorage) FindRemoteStorageClient(p util.FullPath) (client remote_storage.RemoteStorageClient, remoteConf *remote_pb.RemoteConf, found bool) {
 	var storageLocation *remote_pb.RemoteStorageLocation
-	rs.rules.MatchPrefix([]byte(p), func(key []byte, value interface{}) bool {
-		storageLocation = value.(*remote_pb.RemoteStorageLocation)
+	rs.rules.MatchPrefix([]byte(p), func(key []byte, value *remote_pb.RemoteStorageLocation) bool {
+		storageLocation = value
 		return true
 	})
 

weed/filer/stream.go

@@ -69,11 +69,17 @@ func NewFileReader(filerClient filer_pb.FilerClient, entry *filer_pb.Entry) io.R
 
 type DoStreamContent func(writer io.Writer) error
 
-func PrepareStreamContent(masterClient wdclient.HasLookupFileIdFunction, chunks []*filer_pb.FileChunk, offset int64, size int64) (DoStreamContent, error) {
-	return PrepareStreamContentWithThrottler(masterClient, chunks, offset, size, 0)
+func PrepareStreamContent(masterClient wdclient.HasLookupFileIdFunction, jwtFunc VolumeServerJwtFunction, chunks []*filer_pb.FileChunk, offset int64, size int64) (DoStreamContent, error) {
+	return PrepareStreamContentWithThrottler(masterClient, jwtFunc, chunks, offset, size, 0)
 }
 
-func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunction, chunks []*filer_pb.FileChunk, offset int64, size int64, downloadMaxBytesPs int64) (DoStreamContent, error) {
+type VolumeServerJwtFunction func(fileId string) string
+
+func noJwtFunc(string) string {
+	return ""
+}
+
+func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunction, jwtFunc VolumeServerJwtFunction, chunks []*filer_pb.FileChunk, offset int64, size int64, downloadMaxBytesPs int64) (DoStreamContent, error) {
 	glog.V(4).Infof("prepare to stream content for chunks: %d", len(chunks))
 	chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size)
 
@@ -119,7 +125,8 @@ func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunc
 			}
 			urlStrings := fileId2Url[chunkView.FileId]
 			start := time.Now()
-			err := retriedStreamFetchChunkData(writer, urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize))
+			jwt := jwtFunc(chunkView.FileId)
+			err := retriedStreamFetchChunkData(writer, urlStrings, jwt, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize))
 			offset += int64(chunkView.ViewSize)
 			remaining -= int64(chunkView.ViewSize)
 			stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds())
@@ -143,7 +150,7 @@ func PrepareStreamContentWithThrottler(masterClient wdclient.HasLookupFileIdFunc
 }
 
 func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
-	streamFn, err := PrepareStreamContent(masterClient, chunks, offset, size)
+	streamFn, err := PrepareStreamContent(masterClient, noJwtFunc, chunks, offset, size)
 	if err != nil {
 		return err
 	}

weed/ftpd/ftp_server.go

@@ -29,7 +29,7 @@ type SftpServer struct {
 
 var _ = ftpserver.MainDriver(&SftpServer{})
 
-// NewServer returns a new FTP server driver
+// NewFtpServer returns a new FTP server driver
 func NewFtpServer(ftpListener net.Listener, option *FtpServerOption) (*SftpServer, error) {
 	var err error
 	server := &SftpServer{

weed/iamapi/iamapi_management_handlers.go

@@ -3,6 +3,7 @@ package iamapi
 import (
 	"crypto/sha1"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"math/rand"
 	"net/http"
@@ -13,6 +14,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
@@ -424,7 +426,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 	}
 	values := r.PostForm
 	s3cfg := &iam_pb.S3ApiConfiguration{}
-	if err := iama.s3ApiConfig.GetS3ApiConfiguration(s3cfg); err != nil {
+	if err := iama.s3ApiConfig.GetS3ApiConfiguration(s3cfg); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
 		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 		return
 	}

weed/images/cropping.go

@@ -8,7 +8,7 @@ import (
 	"image/png"
 	"io"
 
-	"github.com/disintegration/imaging"
+	"github.com/cognusion/imaging"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 )

weed/images/resizing.go

@@ -8,7 +8,7 @@ import (
 	"image/png"
 	"io"
 
-	"github.com/disintegration/imaging"
+	"github.com/cognusion/imaging"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 

weed/mount/inode_to_path.go

@@ -146,8 +146,8 @@ func (i *InodeToPath) HasPath(path util.FullPath) bool {
 }
 
 func (i *InodeToPath) MarkChildrenCached(fullpath util.FullPath) {
-	i.RLock()
-	defer i.RUnlock()
+	i.Lock()
+	defer i.Unlock()
 	inode, found := i.path2inode[fullpath]
 	if !found {
 		// https://github.com/seaweedfs/seaweedfs/issues/4968

weed/mount/page_writer/page_chunk_mem.go

@@ -63,11 +63,11 @@ func (mc *MemChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64)
 		logicStart := max(off, memChunkBaseOffset+t.StartOffset)
 		logicStop := min(off+int64(len(p)), memChunkBaseOffset+t.stopOffset)
 		if logicStart < logicStop {
+			copy(p[logicStart-off:logicStop-off], mc.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset])
+			maxStop = max(maxStop, logicStop)
+
 			if t.TsNs >= tsNs {
-				copy(p[logicStart-off:logicStop-off], mc.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset])
-				maxStop = max(maxStop, logicStop)
-			} else {
-				println("read old data1", tsNs-t.TsNs, "ns")
+				println("read new data1", t.TsNs - tsNs, "ns")
 			}
 		}
 	}

weed/mount/page_writer/page_chunk_swapfile.go

@@ -129,15 +129,15 @@ func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop in
 		logicStart := max(off, chunkStartOffset+t.StartOffset)
 		logicStop := min(off+int64(len(p)), chunkStartOffset+t.stopOffset)
 		if logicStart < logicStop {
+			actualStart := logicStart - chunkStartOffset + int64(sc.actualChunkIndex)*sc.swapfile.chunkSize
+			if _, err := sc.swapfile.file.ReadAt(p[logicStart-off:logicStop-off], actualStart); err != nil {
+				glog.Errorf("failed to reading swap file %s: %v", sc.swapfile.file.Name(), err)
+				break
+			}
+			maxStop = max(maxStop, logicStop)
+
 			if t.TsNs >= tsNs {
-				actualStart := logicStart - chunkStartOffset + int64(sc.actualChunkIndex)*sc.swapfile.chunkSize
-				if _, err := sc.swapfile.file.ReadAt(p[logicStart-off:logicStop-off], actualStart); err != nil {
-					glog.Errorf("failed to reading swap file %s: %v", sc.swapfile.file.Name(), err)
-					break
-				}
-				maxStop = max(maxStop, logicStop)
-			} else {
-				println("read old data2", tsNs-t.TsNs, "ns")
+				println("read new data2", t.TsNs - tsNs, "ns")
 			}
 		}
 	}

weed/mount/weedfs_attr.go

@@ -145,7 +145,6 @@ func (wfs *WFS) setRootAttr(out *fuse.AttrOut) {
 
 func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry, calculateSize bool) {
 	out.Ino = inode
-	out.Blocks = (out.Size + blockSize - 1) / blockSize
 	setBlksize(out, blockSize)
 	if entry == nil {
 		return
@@ -159,6 +158,7 @@ func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.E
 	if entry.FileMode()&os.ModeSymlink != 0 {
 		out.Size = uint64(len(entry.Attributes.SymlinkTarget))
 	}
+	out.Blocks = (out.Size + blockSize - 1) / blockSize
 	out.Mtime = uint64(entry.Attributes.Mtime)
 	out.Ctime = uint64(entry.Attributes.Mtime)
 	out.Atime = uint64(entry.Attributes.Mtime)

weed/mq/broker/broker_grpc_assign.go

@@ -14,7 +14,6 @@ import (
 // AssignTopicPartitions Runs on the assigned broker, to execute the topic partition assignment
 func (b *MessageQueueBroker) AssignTopicPartitions(c context.Context, request *mq_pb.AssignTopicPartitionsRequest) (*mq_pb.AssignTopicPartitionsResponse, error) {
 	ret := &mq_pb.AssignTopicPartitionsResponse{}
-	self := pb.ServerAddress(fmt.Sprintf("%s:%d", b.option.Ip, b.option.Port))
 
 	// drain existing topic partition subscriptions
 	for _, assignment := range request.BrokerPartitionAssignments {
@@ -23,12 +22,12 @@ func (b *MessageQueueBroker) AssignTopicPartitions(c context.Context, request *m
 		b.accessLock.Lock()
 		if request.IsDraining {
 			// TODO drain existing topic partition subscriptions
-			b.localTopicManager.RemoveTopicPartition(t, partition)
+			b.localTopicManager.RemoveLocalPartition(t, partition)
 		} else {
 			var localPartition *topic.LocalPartition
-			if localPartition = b.localTopicManager.GetTopicPartition(t, partition); localPartition == nil {
-				localPartition = topic.FromPbBrokerPartitionAssignment(self, partition, assignment, b.genLogFlushFunc(t, assignment.Partition), b.genLogOnDiskReadFunc(t, assignment.Partition))
-				b.localTopicManager.AddTopicPartition(t, localPartition)
+			if localPartition = b.localTopicManager.GetLocalPartition(t, partition); localPartition == nil {
+				localPartition = topic.NewLocalPartition(partition, b.genLogFlushFunc(t, assignment.Partition), b.genLogOnDiskReadFunc(t, assignment.Partition))
+				b.localTopicManager.AddLocalPartition(t, localPartition)
 			}
 		}
 		b.accessLock.Unlock()

weed/mq/broker/broker_grpc_configure.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
+	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
@@ -27,6 +28,14 @@ func (b *MessageQueueBroker) ConfigureTopic(ctx context.Context, request *mq_pb.
 		return resp, err
 	}
 
+	// validate the schema
+	if request.RecordType != nil {
+		if _, err = schema.NewSchema(request.RecordType); err != nil {
+			return nil, status.Errorf(codes.InvalidArgument, "invalid record type %+v: %v", request.RecordType, err)
+		}
+	}
+
+
 	t := topic.FromPbTopic(request.Topic)
 	var readErr, assignErr error
 	resp, readErr = b.readTopicConfFromFiler(t)
@@ -56,6 +65,7 @@ func (b *MessageQueueBroker) ConfigureTopic(ctx context.Context, request *mq_pb.
 		return nil, status.Errorf(codes.Unavailable, pub_balancer.ErrNoBroker.Error())
 	}
 	resp.BrokerPartitionAssignments = pub_balancer.AllocateTopicPartitions(b.Balancer.Brokers, request.PartitionCount)
+	resp.RecordType = request.RecordType
 
 	// save the topic configuration on filer
 	if err := b.saveTopicConfToFiler(request.Topic, resp); err != nil {

weed/mq/broker/broker_grpc_pub.go

@@ -5,13 +5,13 @@ import (
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"google.golang.org/grpc/peer"
 	"io"
 	"math/rand"
 	"net"
 	"sync/atomic"
+	"time"
 )
 
 // PUB
@@ -40,73 +40,86 @@ func (b *MessageQueueBroker) PublishMessage(stream mq_pb.SeaweedMessaging_Publis
 	// 2. find the topic metadata owning filer
 	// 3. write to the filer
 
-	var localTopicPartition *topic.LocalPartition
 	req, err := stream.Recv()
 	if err != nil {
 		return err
 	}
 	response := &mq_pb.PublishMessageResponse{}
 	// TODO check whether current broker should be the leader for the topic partition
-	ackInterval := 1
 	initMessage := req.GetInit()
-	var t topic.Topic
-	var p topic.Partition
-	if initMessage != nil {
-		t, p = topic.FromPbTopic(initMessage.Topic), topic.FromPbPartition(initMessage.Partition)
-		localTopicPartition, _, err = b.GetOrGenLocalPartition(t, p)
-		if err != nil {
-			response.Error = fmt.Sprintf("topic %v partition %v not setup", initMessage.Topic, initMessage.Partition)
-			glog.Errorf("topic %v partition %v not setup", initMessage.Topic, initMessage.Partition)
-			return stream.Send(response)
-		}
-		ackInterval = int(initMessage.AckInterval)
-		for _, follower := range initMessage.FollowerBrokers {
-			followErr := b.withBrokerClient(false, pb.ServerAddress(follower), func(client mq_pb.SeaweedMessagingClient) error {
-				_, err := client.PublishFollowMe(context.Background(), &mq_pb.PublishFollowMeRequest{
-					Topic:      initMessage.Topic,
-					Partition:  initMessage.Partition,
-					BrokerSelf: string(b.option.BrokerAddress()),
-				})
-				return err
-			})
-			if followErr != nil {
-				response.Error = fmt.Sprintf("follower %v failed: %v", follower, followErr)
-				glog.Errorf("follower %v failed: %v", follower, followErr)
-				return stream.Send(response)
-			}
-		}
-		stream.Send(response)
-	} else {
+	if initMessage == nil {
 		response.Error = fmt.Sprintf("missing init message")
 		glog.Errorf("missing init message")
 		return stream.Send(response)
 	}
 
+	// get or generate a local partition
+	t, p := topic.FromPbTopic(initMessage.Topic), topic.FromPbPartition(initMessage.Partition)
+	localTopicPartition, getOrGenErr := b.GetOrGenerateLocalPartition(t, p)
+	if getOrGenErr != nil {
+		response.Error = fmt.Sprintf("topic %v not found: %v", t, getOrGenErr)
+		glog.Errorf("topic %v not found: %v", t, getOrGenErr)
+		return stream.Send(response)
+	}
+
+	// connect to follower brokers
+	if followerErr := localTopicPartition.MaybeConnectToFollowers(initMessage, b.grpcDialOption); followerErr != nil {
+		response.Error = followerErr.Error()
+		glog.Errorf("MaybeConnectToFollowers: %v", followerErr)
+		return stream.Send(response)
+	}
+
+	var receivedSequence, acknowledgedSequence  int64
+	var isClosed bool
+
+	// start sending ack to publisher
+	ackInterval := int64(1)
+	if initMessage.AckInterval > 0 {
+		ackInterval = int64(initMessage.AckInterval)
+	}
+	go func() {
+		defer func() {
+			// println("stop sending ack to publisher", initMessage.PublisherName)
+		}()
+
+		lastAckTime := time.Now()
+		for !isClosed {
+			receivedSequence = atomic.LoadInt64(&localTopicPartition.AckTsNs)
+			if acknowledgedSequence < receivedSequence && (receivedSequence - acknowledgedSequence >= ackInterval || time.Since(lastAckTime) > 1*time.Second){
+				acknowledgedSequence = receivedSequence
+				response := &mq_pb.PublishMessageResponse{
+					AckSequence: acknowledgedSequence,
+				}
+				if err := stream.Send(response); err != nil {
+					glog.Errorf("Error sending response %v: %v", response, err)
+				}
+				// println("sent ack", acknowledgedSequence, "=>", initMessage.PublisherName)
+				lastAckTime = time.Now()
+			} else {
+				time.Sleep(1 * time.Second)
+			}
+		}
+	}()
+
+
+	// process each published messages
 	clientName := fmt.Sprintf("%v-%4d/%s/%v", findClientAddress(stream.Context()), rand.Intn(10000), initMessage.Topic, initMessage.Partition)
 	localTopicPartition.Publishers.AddPublisher(clientName, topic.NewLocalPublisher())
 
-	ackCounter := 0
-	var ackSequence int64
-	var isStopping int32
-	respChan := make(chan *mq_pb.PublishMessageResponse, 128)
 	defer func() {
-		atomic.StoreInt32(&isStopping, 1)
-		respChan <- &mq_pb.PublishMessageResponse{
-			AckSequence: ackSequence,
-		}
-		close(respChan)
+		// remove the publisher
 		localTopicPartition.Publishers.RemovePublisher(clientName)
-		glog.V(0).Infof("topic %v partition %v published %d messges Publisher:%d Subscriber:%d", initMessage.Topic, initMessage.Partition, ackSequence, localTopicPartition.Publishers.Size(), localTopicPartition.Subscribers.Size())
 		if localTopicPartition.MaybeShutdownLocalPartition() {
-			b.localTopicManager.RemoveTopicPartition(t, p)
+			b.localTopicManager.RemoveLocalPartition(t, p)
+			glog.V(0).Infof("Removed local topic %v partition %v", initMessage.Topic, initMessage.Partition)
 		}
 	}()
-	go func() {
-		for resp := range respChan {
-			if err := stream.Send(resp); err != nil {
-				glog.Errorf("Error sending response %v: %v", resp, err)
-			}
-		}
+
+	// send a hello message
+	stream.Send(&mq_pb.PublishMessageResponse{})
+
+	defer func() {
+		isClosed = true
 	}()
 
 	// process each published messages
@@ -117,28 +130,26 @@ func (b *MessageQueueBroker) PublishMessage(stream mq_pb.SeaweedMessaging_Publis
 			if err == io.EOF {
 				break
 			}
-			glog.V(0).Infof("topic %v partition %v publish stream error: %v", initMessage.Topic, initMessage.Partition, err)
-			return err
+			glog.V(0).Infof("topic %v partition %v publish stream from %s error: %v", initMessage.Topic, initMessage.Partition, initMessage.PublisherName, err)
+			break
 		}
 
 		// Process the received message
-		if dataMessage := req.GetData(); dataMessage != nil {
-			localTopicPartition.Publish(dataMessage)
+		dataMessage := req.GetData()
+		if dataMessage == nil {
+			continue
 		}
 
-		ackCounter++
-		ackSequence++
-		if ackCounter >= ackInterval {
-			ackCounter = 0
-			// send back the ack
-			response := &mq_pb.PublishMessageResponse{
-				AckSequence: ackSequence,
-			}
-			respChan <- response
+		// The control message should still be sent to the follower
+		// to avoid timing issue when ack messages.
+
+		// send to the local partition
+		if err = localTopicPartition.Publish(dataMessage); err != nil {
+			return fmt.Errorf("topic %v partition %v publish error: %v", initMessage.Topic, initMessage.Partition, err)
 		}
 	}
 
-	glog.V(0).Infof("topic %v partition %v publish stream closed.", initMessage.Topic, initMessage.Partition)
+	glog.V(0).Infof("topic %v partition %v publish stream from %s closed.", initMessage.Topic, initMessage.Partition, initMessage.PublisherName)
 
 	return nil
 }

weed/mq/broker/broker_grpc_pub_follow.go

@@ -1,96 +1,151 @@
 package broker
 
 import (
-	"context"
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util/buffered_queue"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
 	"io"
-	"math/rand"
-	"sync"
 	"time"
 )
 
-func (b *MessageQueueBroker) PublishFollowMe(c context.Context, request *mq_pb.PublishFollowMeRequest) (*mq_pb.PublishFollowMeResponse, error) {
-	glog.V(0).Infof("PublishFollowMe %v", request)
-	var wg sync.WaitGroup
-	wg.Add(1)
-	var ret error
-	go b.withBrokerClient(true, pb.ServerAddress(request.BrokerSelf), func(client mq_pb.SeaweedMessagingClient) error {
-		followerId := rand.Int31()
-		subscribeClient, err := client.FollowInMemoryMessages(context.Background(), &mq_pb.FollowInMemoryMessagesRequest{
-			Message: &mq_pb.FollowInMemoryMessagesRequest_Init{
-				Init: &mq_pb.FollowInMemoryMessagesRequest_InitMessage{
-					ConsumerGroup: string(b.option.BrokerAddress()),
-					ConsumerId:    fmt.Sprintf("followMe-%d", followerId),
-					FollowerId:    followerId,
-					Topic:         request.Topic,
-					PartitionOffset: &mq_pb.PartitionOffset{
-						Partition: request.Partition,
-						StartTsNs: 0,
-						StartType: mq_pb.PartitionOffsetStartType_EARLIEST_IN_MEMORY,
-					},
-				},
-			},
-		})
-
-		if err != nil {
-			glog.Errorf("FollowInMemoryMessages error: %v", err)
-			ret = err
-			return err
-		}
-
-		// receive first hello message
-		resp, err := subscribeClient.Recv()
-		if err != nil {
-			return fmt.Errorf("FollowInMemoryMessages recv first message error: %v", err)
-		}
-		if resp == nil {
-			glog.V(0).Infof("doFollowInMemoryMessage recv first message nil response")
-			return io.ErrUnexpectedEOF
-		}
-		wg.Done()
-
-		b.doFollowInMemoryMessage(context.Background(), subscribeClient)
-
-		return nil
-	})
-	wg.Wait()
-	return &mq_pb.PublishFollowMeResponse{}, ret
+type memBuffer struct {
+	buf        []byte
+	startTime  time.Time
+	stopTime   time.Time
 }
+func (b *MessageQueueBroker) PublishFollowMe(stream mq_pb.SeaweedMessaging_PublishFollowMeServer) (err error) {
+	var req *mq_pb.PublishFollowMeRequest
+	req, err = stream.Recv()
+	if err != nil {
+		return err
+	}
+	initMessage := req.GetInit()
+	if initMessage == nil {
+		return fmt.Errorf("missing init message")
+	}
 
-func (b *MessageQueueBroker) doFollowInMemoryMessage(c context.Context, client mq_pb.SeaweedMessaging_FollowInMemoryMessagesClient) {
+	// create an in-memory queue of buffered messages
+	inMemoryBuffers := buffered_queue.NewBufferedQueue[memBuffer](4)
+	logBuffer := b.buildFollowerLogBuffer(inMemoryBuffers)
+
+	lastFlushTsNs := time.Now().UnixNano()
+
+	// follow each published messages
 	for {
-		resp, err := client.Recv()
+		// receive a message
+		req, err = stream.Recv()
 		if err != nil {
-			if err != io.EOF {
-				glog.V(0).Infof("doFollowInMemoryMessage error: %v", err)
+			if err == io.EOF {
+				err = nil
+				break
 			}
-			return
+			glog.V(0).Infof("topic %v partition %v publish stream error: %v", initMessage.Topic, initMessage.Partition, err)
+			break
 		}
-		if resp == nil {
-			glog.V(0).Infof("doFollowInMemoryMessage nil response")
-			return
-		}
-		if resp.Message != nil {
-			// process ctrl message or data message
-			switch m := resp.Message.(type) {
-			case *mq_pb.FollowInMemoryMessagesResponse_Data:
-				// process data message
-				print("d")
-			case *mq_pb.FollowInMemoryMessagesResponse_Ctrl:
-				// process ctrl message
-				if m.Ctrl.FlushedSequence > 0 {
-					flushTime := time.Unix(0, m.Ctrl.FlushedSequence)
-					glog.V(0).Infof("doFollowInMemoryMessage flushTime: %v", flushTime)
-				}
-				if m.Ctrl.FollowerChangedToId != 0 {
-					// follower changed
-					glog.V(0).Infof("doFollowInMemoryMessage follower changed to %d", m.Ctrl.FollowerChangedToId)
-					return
+
+		// Process the received message
+		if dataMessage := req.GetData(); dataMessage != nil {
+
+			// TODO: change this to DataMessage
+			// log the message
+			logBuffer.AddToBuffer(dataMessage)
+
+			// send back the ack
+			if err := stream.Send(&mq_pb.PublishFollowMeResponse{
+				AckTsNs: dataMessage.TsNs,
+			}); err != nil {
+				glog.Errorf("Error sending response %v: %v", dataMessage, err)
+			}
+			// println("ack", string(dataMessage.Key), dataMessage.TsNs)
+		} else if closeMessage := req.GetClose(); closeMessage != nil {
+			glog.V(0).Infof("topic %v partition %v publish stream closed: %v", initMessage.Topic, initMessage.Partition, closeMessage)
+			break
+		} else if flushMessage := req.GetFlush(); flushMessage != nil {
+			glog.V(0).Infof("topic %v partition %v publish stream flushed: %v", initMessage.Topic, initMessage.Partition, flushMessage)
+
+			lastFlushTsNs = flushMessage.TsNs
+
+			// drop already flushed messages
+			for mem, found := inMemoryBuffers.PeekHead(); found; mem, found = inMemoryBuffers.PeekHead() {
+				if mem.stopTime.UnixNano() <= flushMessage.TsNs {
+					inMemoryBuffers.Dequeue()
+					// println("dropping flushed messages: ", mem.startTime.UnixNano(), mem.stopTime.UnixNano(), len(mem.buf))
+				} else {
+					break
 				}
 			}
+
+		} else {
+			glog.Errorf("unknown message: %v", req)
 		}
 	}
+
+
+	t, p := topic.FromPbTopic(initMessage.Topic), topic.FromPbPartition(initMessage.Partition)
+
+	logBuffer.ShutdownLogBuffer()
+	// wait until all messages are sent to inMemoryBuffers
+	for !logBuffer.IsAllFlushed() {
+		time.Sleep(113 * time.Millisecond)
+	}
+
+	topicDir := fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
+	partitionGeneration := time.Unix(0, p.UnixTimeNs).UTC().Format(topic.TIME_FORMAT)
+	partitionDir := fmt.Sprintf("%s/%s/%04d-%04d", topicDir, partitionGeneration, p.RangeStart, p.RangeStop)
+
+
+	// flush the remaining messages
+	inMemoryBuffers.CloseInput()
+	for mem, found := inMemoryBuffers.Dequeue(); found; mem, found = inMemoryBuffers.Dequeue() {
+		if len(mem.buf) == 0 {
+			continue
+		}
+
+		startTime, stopTime := mem.startTime.UTC(), mem.stopTime.UTC()
+
+		if stopTime.UnixNano() <= lastFlushTsNs {
+			glog.V(0).Infof("dropping remaining data at %v %v", t, p)
+			continue
+		}
+
+		// TODO trim data earlier than lastFlushTsNs
+
+		targetFile := fmt.Sprintf("%s/%s", partitionDir, startTime.Format(topic.TIME_FORMAT))
+
+		for {
+			if err := b.appendToFile(targetFile, mem.buf); err != nil {
+				glog.V(0).Infof("metadata log write failed %s: %v", targetFile, err)
+				time.Sleep(737 * time.Millisecond)
+			} else {
+				break
+			}
+		}
+
+		glog.V(0).Infof("flushed remaining data at %v to %s size %d", mem.stopTime.UnixNano(), targetFile, len(mem.buf))
+	}
+
+	glog.V(0).Infof("shut down follower for %v %v", t, p)
+
+	return err
+}
+
+func (b *MessageQueueBroker) buildFollowerLogBuffer(inMemoryBuffers *buffered_queue.BufferedQueue[memBuffer]) *log_buffer.LogBuffer {
+	lb := log_buffer.NewLogBuffer("follower",
+		2*time.Minute, func(logBuffer *log_buffer.LogBuffer, startTime, stopTime time.Time, buf []byte) {
+			if len(buf) == 0 {
+				return
+			}
+			inMemoryBuffers.Enqueue(memBuffer{
+				buf:       buf,
+				startTime: startTime,
+				stopTime:  stopTime,
+			})
+			glog.V(0).Infof("queue up %d~%d size %d", startTime.UnixNano(), stopTime.UnixNano(), len(buf))
+		}, nil, func() {
+		})
+	return lb
 }

weed/mq/broker/broker_grpc_sub.go

@@ -8,7 +8,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
-	"sync/atomic"
 	"time"
 )
 
@@ -17,40 +16,20 @@ func (b *MessageQueueBroker) SubscribeMessage(req *mq_pb.SubscribeMessageRequest
 	ctx := stream.Context()
 	clientName := fmt.Sprintf("%s/%s-%s", req.GetInit().ConsumerGroup, req.GetInit().ConsumerId, req.GetInit().ClientId)
 
+	initMessage := req.GetInit()
+	if initMessage == nil {
+		glog.Errorf("missing init message")
+		return fmt.Errorf("missing init message")
+	}
+
 	t := topic.FromPbTopic(req.GetInit().Topic)
 	partition := topic.FromPbPartition(req.GetInit().GetPartitionOffset().GetPartition())
 
 	glog.V(0).Infof("Subscriber %s on %v %v connected", req.GetInit().ConsumerId, t, partition)
 
-	waitIntervalCount := 0
-
-	var localTopicPartition *topic.LocalPartition
-	for localTopicPartition == nil {
-		localTopicPartition, _, err = b.GetOrGenLocalPartition(t, partition)
-		if err != nil {
-			glog.V(1).Infof("topic %v partition %v not setup", t, partition)
-		}
-		if localTopicPartition != nil {
-			break
-		}
-		waitIntervalCount++
-		if waitIntervalCount > 10 {
-			waitIntervalCount = 10
-		}
-		time.Sleep(time.Duration(waitIntervalCount) * 337 * time.Millisecond)
-		// Check if the client has disconnected by monitoring the context
-		select {
-		case <-ctx.Done():
-			err := ctx.Err()
-			if err == context.Canceled {
-				// Client disconnected
-				return nil
-			}
-			glog.V(0).Infof("Subscriber %s disconnected: %v", clientName, err)
-			return nil
-		default:
-			// Continue processing the request
-		}
+	localTopicPartition, getOrGenErr := b.GetOrGenerateLocalPartition(t, partition)
+	if getOrGenErr != nil {
+		return getOrGenErr
 	}
 
 	localTopicPartition.Subscribers.AddSubscriber(clientName, topic.NewLocalSubscriber())
@@ -64,7 +43,7 @@ func (b *MessageQueueBroker) SubscribeMessage(req *mq_pb.SubscribeMessageRequest
 		localTopicPartition.Subscribers.RemoveSubscriber(clientName)
 		glog.V(0).Infof("Subscriber %s on %v %v disconnected, sent %d", clientName, t, partition, counter)
 		if localTopicPartition.MaybeShutdownLocalPartition() {
-			b.localTopicManager.RemoveTopicPartition(t, partition)
+			b.localTopicManager.RemoveLocalPartition(t, partition)
 		}
 	}()
 
@@ -129,174 +108,3 @@ func getRequestPosition(offset *mq_pb.PartitionOffset) (startPosition log_buffer
 	}
 	return
 }
-
-func (b *MessageQueueBroker) FollowInMemoryMessages(req *mq_pb.FollowInMemoryMessagesRequest, stream mq_pb.SeaweedMessaging_FollowInMemoryMessagesServer) (err error) {
-	ctx := stream.Context()
-	clientName := req.GetInit().ConsumerId
-
-	t := topic.FromPbTopic(req.GetInit().Topic)
-	partition := topic.FromPbPartition(req.GetInit().GetPartitionOffset().GetPartition())
-
-	glog.V(0).Infof("FollowInMemoryMessages %s on %v %v connected", clientName, t, partition)
-
-	waitIntervalCount := 0
-
-	var localTopicPartition *topic.LocalPartition
-	for localTopicPartition == nil {
-		localTopicPartition, _, err = b.GetOrGenLocalPartition(t, partition)
-		if err != nil {
-			glog.V(1).Infof("topic %v partition %v not setup", t, partition)
-		}
-		if localTopicPartition != nil {
-			break
-		}
-		waitIntervalCount++
-		if waitIntervalCount > 32 {
-			waitIntervalCount = 32
-		}
-		time.Sleep(time.Duration(waitIntervalCount) * 137 * time.Millisecond)
-		// Check if the client has disconnected by monitoring the context
-		select {
-		case <-ctx.Done():
-			err := ctx.Err()
-			if err == context.Canceled {
-				// Client disconnected
-				return nil
-			}
-			glog.V(0).Infof("FollowInMemoryMessages %s disconnected: %v", clientName, err)
-			return nil
-		default:
-			// Continue processing the request
-		}
-	}
-
-	// set the current follower id
-	followerId := req.GetInit().FollowerId
-	atomic.StoreInt32(&localTopicPartition.FollowerId, followerId)
-
-	glog.V(0).Infof("FollowInMemoryMessages %s connected on %v %v", clientName, t, partition)
-	isConnected := true
-	sleepIntervalCount := 0
-
-	var counter int64
-	defer func() {
-		isConnected = false
-		glog.V(0).Infof("FollowInMemoryMessages %s on %v %v disconnected, sent %d", clientName, t, partition, counter)
-	}()
-
-	// send first hello message
-	// to indicate the follower is connected
-	stream.Send(&mq_pb.FollowInMemoryMessagesResponse{
-		Message: &mq_pb.FollowInMemoryMessagesResponse_Ctrl{
-			Ctrl: &mq_pb.FollowInMemoryMessagesResponse_CtrlMessage{},
-		},
-	})
-
-	var startPosition log_buffer.MessagePosition
-	if req.GetInit() != nil && req.GetInit().GetPartitionOffset() != nil {
-		startPosition = getRequestPosition(req.GetInit().GetPartitionOffset())
-	}
-
-	var prevFlushTsNs int64
-
-	_, _, err = localTopicPartition.LogBuffer.LoopProcessLogData(clientName, startPosition, 0, func() bool {
-		if !isConnected {
-			return false
-		}
-		sleepIntervalCount++
-		if sleepIntervalCount > 32 {
-			sleepIntervalCount = 32
-		}
-		time.Sleep(time.Duration(sleepIntervalCount) * 137 * time.Millisecond)
-
-		if localTopicPartition.LogBuffer.IsStopping() {
-			newFollowerId := atomic.LoadInt32(&localTopicPartition.FollowerId)
-			glog.V(0).Infof("FollowInMemoryMessages1 %s on %v %v follower id changed to %d", clientName, t, partition, newFollowerId)
-			stream.Send(&mq_pb.FollowInMemoryMessagesResponse{
-				Message: &mq_pb.FollowInMemoryMessagesResponse_Ctrl{
-					Ctrl: &mq_pb.FollowInMemoryMessagesResponse_CtrlMessage{
-						FollowerChangedToId: newFollowerId,
-					},
-				},
-			})
-			return false
-		}
-
-		// Check if the client has disconnected by monitoring the context
-		select {
-		case <-ctx.Done():
-			err := ctx.Err()
-			if err == context.Canceled {
-				// Client disconnected
-				return false
-			}
-			glog.V(0).Infof("Subscriber %s disconnected: %v", clientName, err)
-			return false
-		default:
-			// Continue processing the request
-		}
-
-		// send the last flushed sequence
-		flushTsNs := atomic.LoadInt64(&localTopicPartition.LogBuffer.LastFlushTsNs)
-		if flushTsNs != prevFlushTsNs {
-			prevFlushTsNs = flushTsNs
-			stream.Send(&mq_pb.FollowInMemoryMessagesResponse{
-				Message: &mq_pb.FollowInMemoryMessagesResponse_Ctrl{
-					Ctrl: &mq_pb.FollowInMemoryMessagesResponse_CtrlMessage{
-						FlushedSequence: flushTsNs,
-					},
-				},
-			})
-		}
-
-		return true
-	}, func(logEntry *filer_pb.LogEntry) (bool, error) {
-		// reset the sleep interval count
-		sleepIntervalCount = 0
-
-		// check the follower id
-		newFollowerId := atomic.LoadInt32(&localTopicPartition.FollowerId)
-		if newFollowerId != followerId {
-			glog.V(0).Infof("FollowInMemoryMessages2 %s on %v %v follower id %d changed to %d", clientName, t, partition, followerId, newFollowerId)
-			stream.Send(&mq_pb.FollowInMemoryMessagesResponse{
-				Message: &mq_pb.FollowInMemoryMessagesResponse_Ctrl{
-					Ctrl: &mq_pb.FollowInMemoryMessagesResponse_CtrlMessage{
-						FollowerChangedToId: newFollowerId,
-					},
-				},
-			})
-			return true, nil
-		}
-
-		// send the last flushed sequence
-		flushTsNs := atomic.LoadInt64(&localTopicPartition.LogBuffer.LastFlushTsNs)
-		if flushTsNs != prevFlushTsNs {
-			prevFlushTsNs = flushTsNs
-			stream.Send(&mq_pb.FollowInMemoryMessagesResponse{
-				Message: &mq_pb.FollowInMemoryMessagesResponse_Ctrl{
-					Ctrl: &mq_pb.FollowInMemoryMessagesResponse_CtrlMessage{
-						FlushedSequence: flushTsNs,
-					},
-				},
-			})
-		}
-
-		// send the log entry
-		if err := stream.Send(&mq_pb.FollowInMemoryMessagesResponse{
-			Message: &mq_pb.FollowInMemoryMessagesResponse_Data{
-				Data: &mq_pb.DataMessage{
-					Key:   logEntry.Key,
-					Value: logEntry.Data,
-					TsNs:  logEntry.TsNs,
-				},
-			}}); err != nil {
-			glog.Errorf("Error sending setup response: %v", err)
-			return false, err
-		}
-
-		counter++
-		return false, nil
-	})
-
-	return err
-}

weed/mq/broker/broker_grpc_sub_coordinator.go

@@ -39,18 +39,11 @@ func (b *MessageQueueBroker) SubscriberToSubCoordinator(stream mq_pb.SeaweedMess
 	go func() {
 		// try to load the partition assignment from filer
 		if conf, err := b.readTopicConfFromFiler(topic.FromPbTopic(initMessage.Topic)); err == nil {
-			assignedPartitions := make([]*mq_pb.SubscriberToSubCoordinatorResponse_AssignedPartition, len(conf.BrokerPartitionAssignments))
-			for i, assignment := range conf.BrokerPartitionAssignments {
-				assignedPartitions[i] = &mq_pb.SubscriberToSubCoordinatorResponse_AssignedPartition{
-					Partition: assignment.Partition,
-					Broker:    assignment.LeaderBroker,
-				}
-			}
 			// send partition assignment to subscriber
 			cgi.ResponseChan <- &mq_pb.SubscriberToSubCoordinatorResponse{
 				Message: &mq_pb.SubscriberToSubCoordinatorResponse_Assignment_{
 					Assignment: &mq_pb.SubscriberToSubCoordinatorResponse_Assignment{
-						AssignedPartitions: assignedPartitions,
+						PartitionAssignments: conf.BrokerPartitionAssignments,
 					},
 				},
 			}

weed/mq/broker/broker_test.go

@@ -1,47 +0,0 @@
-package broker
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/test/bufconn"
-)
-
-var lis *bufconn.Listener
-
-func init() {
-	lis = bufconn.Listen(1024 * 1024)
-	server := grpc.NewServer()
-	mq_pb.RegisterSeaweedMessagingServer(server, &MessageQueueBroker{})
-	go func() {
-		if err := server.Serve(lis); err != nil {
-			fmt.Printf("Server exited with error: %v", err)
-		}
-	}()
-}
-
-func bufDialer(string, time.Duration) (net.Conn, error) {
-	return lis.Dial()
-}
-
-func TestMessageQueueBroker_ListTopics(t *testing.T) {
-	conn, err := grpc.DialContext(context.Background(), "bufnet", grpc.WithDialer(bufDialer), grpc.WithInsecure())
-	if err != nil {
-		t.Fatalf("Failed to dial bufnet: %v", err)
-	}
-	defer conn.Close()
-
-	client := mq_pb.NewSeaweedMessagingClient(conn)
-	request := &mq_pb.ListTopicsRequest{}
-
-	_, err = client.ListTopics(context.Background(), request)
-	if err == nil {
-		t.Fatalf("Add failed: %v", err)
-	}
-
-}

weed/mq/broker/broker_topic_conf_read_write.go

@@ -56,12 +56,27 @@ func (b *MessageQueueBroker) readTopicConfFromFiler(t topic.Topic) (conf *mq_pb.
 	return conf, nil
 }
 
-func (b *MessageQueueBroker) GetOrGenLocalPartition(t topic.Topic, partition topic.Partition) (localPartition *topic.LocalPartition, isGenerated bool, err error) {
+func (b *MessageQueueBroker) GetOrGenerateLocalPartition(t topic.Topic, partition topic.Partition) (localTopicPartition *topic.LocalPartition, getOrGenError error) {
+	// get or generate a local partition
+	conf, readConfErr := b.readTopicConfFromFiler(t)
+	if readConfErr != nil {
+		glog.Errorf("topic %v not found: %v", t, readConfErr)
+		return nil, fmt.Errorf("topic %v not found: %v", t, readConfErr)
+	}
+	localTopicPartition, _, getOrGenError = b.doGetOrGenLocalPartition(t, partition, conf)
+	if getOrGenError != nil {
+		glog.Errorf("topic %v partition %v not setup: %v", t, partition, getOrGenError)
+		return nil, fmt.Errorf("topic %v partition %v not setup: %v", t, partition, getOrGenError)
+	}
+	return localTopicPartition, nil
+}
+
+func (b *MessageQueueBroker) doGetOrGenLocalPartition(t topic.Topic, partition topic.Partition, conf *mq_pb.ConfigureTopicResponse) (localPartition *topic.LocalPartition, isGenerated bool, err error) {
 	b.accessLock.Lock()
 	defer b.accessLock.Unlock()
 
-	if localPartition = b.localTopicManager.GetTopicPartition(t, partition); localPartition == nil {
-		localPartition, isGenerated, err = b.genLocalPartitionFromFiler(t, partition)
+	if localPartition = b.localTopicManager.GetLocalPartition(t, partition); localPartition == nil {
+		localPartition, isGenerated, err = b.genLocalPartitionFromFiler(t, partition, conf)
 		if err != nil {
 			return nil, false, err
 		}
@@ -69,16 +84,12 @@ func (b *MessageQueueBroker) GetOrGenLocalPartition(t topic.Topic, partition top
 	return localPartition, isGenerated, nil
 }
 
-func (b *MessageQueueBroker) genLocalPartitionFromFiler(t topic.Topic, partition topic.Partition) (localPartition *topic.LocalPartition, isGenerated bool, err error) {
+func (b *MessageQueueBroker) genLocalPartitionFromFiler(t topic.Topic, partition topic.Partition, conf *mq_pb.ConfigureTopicResponse) (localPartition *topic.LocalPartition, isGenerated bool, err error) {
 	self := b.option.BrokerAddress()
-	conf, err := b.readTopicConfFromFiler(t)
-	if err != nil {
-		return nil, isGenerated, err
-	}
 	for _, assignment := range conf.BrokerPartitionAssignments {
 		if assignment.LeaderBroker == string(self) && partition.Equals(topic.FromPbPartition(assignment.Partition)) {
-			localPartition = topic.FromPbBrokerPartitionAssignment(b.option.BrokerAddress(), partition, assignment, b.genLogFlushFunc(t, assignment.Partition), b.genLogOnDiskReadFunc(t, assignment.Partition))
-			b.localTopicManager.AddTopicPartition(t, localPartition)
+			localPartition = topic.NewLocalPartition(partition, b.genLogFlushFunc(t, assignment.Partition), b.genLogOnDiskReadFunc(t, assignment.Partition))
+			b.localTopicManager.AddLocalPartition(t, localPartition)
 			isGenerated = true
 			break
 		}

weed/mq/broker/broker_topic_partition_read_write.go

@@ -41,6 +41,16 @@ func (b *MessageQueueBroker) genLogFlushFunc(t topic.Topic, partition *mq_pb.Par
 		}
 
 		atomic.StoreInt64(&logBuffer.LastFlushTsNs, stopTime.UnixNano())
+
+		b.accessLock.Lock()
+		defer b.accessLock.Unlock()
+		p := topic.FromPbPartition(partition)
+		if localPartition:=b.localTopicManager.GetLocalPartition(t, p); localPartition!=nil {
+			localPartition.NotifyLogFlushed(logBuffer.LastFlushTsNs)
+		}
+
+		println("flushing at", logBuffer.LastFlushTsNs, "to", targetFile, "size", len(buf))
+
 	}
 }
 

weed/mq/client/cmd/weed_pub_kv/publisher_kv.go

@@ -16,6 +16,8 @@ var (
 	concurrency    = flag.Int("c", 4, "concurrent publishers")
 	partitionCount = flag.Int("p", 6, "partition count")
 
+	clientName   = flag.String("client", "c1", "client name")
+
 	namespace   = flag.String("ns", "test", "namespace")
 	t           = flag.String("t", "test", "t")
 	seedBrokers = flag.String("brokers", "localhost:17777", "seed brokers")
@@ -25,25 +27,30 @@ func doPublish(publisher *pub_client.TopicPublisher, id int) {
 	startTime := time.Now()
 	for i := 0; i < *messageCount / *concurrency; i++ {
 		// Simulate publishing a message
-		key := []byte(fmt.Sprintf("key-%d-%d", id, i))
-		value := []byte(fmt.Sprintf("value-%d-%d", id, i))
+		key := []byte(fmt.Sprintf("key-%s-%d-%d", *clientName, id, i))
+		value := []byte(fmt.Sprintf("value-%s-%d-%d", *clientName, id, i))
 		if err := publisher.Publish(key, value); err != nil {
 			fmt.Println(err)
 			break
 		}
+		time.Sleep(time.Second)
 		// println("Published", string(key), string(value))
 	}
+	if err := publisher.FinishPublish(); err != nil {
+		fmt.Println(err)
+	}
 	elapsed := time.Since(startTime)
-	log.Printf("Publisher %d finished in %s", id, elapsed)
+	log.Printf("Publisher %s-%d finished in %s", *clientName, id, elapsed)
 }
 
 func main() {
 	flag.Parse()
+
 	config := &pub_client.PublisherConfiguration{
-		Topic:                     topic.NewTopic(*namespace, *t),
-		CreateTopic:               true,
-		CreateTopicPartitionCount: int32(*partitionCount),
-		Brokers:                   strings.Split(*seedBrokers, ","),
+		Topic:          topic.NewTopic(*namespace, *t),
+		PartitionCount: int32(*partitionCount),
+		Brokers:        strings.Split(*seedBrokers, ","),
+		PublisherName:  *clientName,
 	}
 	publisher := pub_client.NewTopicPublisher(config)
 

weed/mq/client/cmd/weed_pub_record/publisher_record.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/mq/client/pub_client"
+	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"log"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	messageCount   = flag.Int("n", 1000, "message count")
+	messageDelay   = flag.Duration("d", time.Second, "delay between messages")
+	concurrency    = flag.Int("c", 4, "concurrent publishers")
+	partitionCount = flag.Int("p", 6, "partition count")
+
+	clientName   = flag.String("client", "c1", "client name")
+
+	namespace   = flag.String("ns", "test", "namespace")
+	t           = flag.String("t", "test", "t")
+	seedBrokers = flag.String("brokers", "localhost:17777", "seed brokers")
+)
+
+func doPublish(publisher *pub_client.TopicPublisher, id int) {
+	startTime := time.Now()
+	for i := 0; i < *messageCount / *concurrency; i++ {
+		// Simulate publishing a message
+		myRecord := genMyRecord(int32(i))
+		if err := publisher.PublishRecord(myRecord.Key, myRecord.ToRecordValue()); err != nil {
+			fmt.Println(err)
+			break
+		}
+		if *messageDelay > 0 {
+			time.Sleep(*messageDelay)
+			fmt.Printf("sent %+v\n", myRecord)
+		}
+	}
+	if err := publisher.FinishPublish(); err != nil {
+		fmt.Println(err)
+	}
+	elapsed := time.Since(startTime)
+	log.Printf("Publisher %s-%d finished in %s", *clientName, id, elapsed)
+}
+
+type MyRecord struct {
+	Key    []byte
+	Field1 []byte
+	Field2 string
+	Field3 int32
+	Field4 int64
+	Field5 float32
+	Field6 float64
+	Field7 bool
+}
+
+func genMyRecord(id int32) *MyRecord {
+	return &MyRecord{
+		Key:    []byte(fmt.Sprintf("key-%s-%d", *clientName, id)),
+		Field1: []byte(fmt.Sprintf("field1-%s-%d", *clientName, id)),
+		Field2: fmt.Sprintf("field2-%s-%d", *clientName, id),
+		Field3: id,
+		Field4: int64(id),
+		Field5: float32(id),
+		Field6: float64(id),
+		Field7: id%2 == 0,
+	}
+}
+
+func (r *MyRecord) ToRecordValue() *schema_pb.RecordValue {
+	return schema.RecordBegin().
+		SetBytes("key", r.Key).
+		SetBytes("field1", r.Field1).
+		SetString("field2", r.Field2).
+		SetInt32("field3", r.Field3).
+		SetInt64("field4", r.Field4).
+		SetFloat("field5", r.Field5).
+		SetDouble("field6", r.Field6).
+		SetBool("field7", r.Field7).
+		RecordEnd()
+}
+
+func main() {
+	flag.Parse()
+
+	recordType := schema.RecordTypeBegin().
+		WithField("key", schema.TypeBytes).
+		WithField("field1", schema.TypeBytes).
+		WithField("field2", schema.TypeString).
+		WithField("field3", schema.TypeInt32).
+		WithField("field4", schema.TypeInt64).
+		WithField("field5", schema.TypeFloat).
+		WithField("field6", schema.TypeDouble).
+		WithField("field7", schema.TypeBoolean).
+		RecordTypeEnd()
+
+	config := &pub_client.PublisherConfiguration{
+		Topic:          topic.NewTopic(*namespace, *t),
+		PartitionCount: int32(*partitionCount),
+		Brokers:        strings.Split(*seedBrokers, ","),
+		PublisherName:  *clientName,
+		RecordType:     recordType,
+	}
+	publisher := pub_client.NewTopicPublisher(config)
+
+	startTime := time.Now()
+
+	var wg sync.WaitGroup
+	// Start multiple publishers
+	for i := 0; i < *concurrency; i++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+			doPublish(publisher, id)
+		}(i)
+	}
+
+	// Wait for all publishers to finish
+	wg.Wait()
+	elapsed := time.Since(startTime)
+	publisher.Shutdown()
+
+	log.Printf("Published %d messages in %s (%.2f msg/s)", *messageCount, elapsed, float64(*messageCount)/elapsed.Seconds())
+
+}

weed/mq/client/cmd/weed_sub_kv/subscriber_kv.go

@@ -52,7 +52,7 @@ func main() {
 	})
 
 	subscriber.SetCompletionFunc(func() {
-		glog.V(0).Infof("done recived %d messages", counter)
+		glog.V(0).Infof("done received %d messages", counter)
 	})
 
 	if err := subscriber.Subscribe(); err != nil {

weed/mq/client/cmd/weed_sub_record/subscriber_record.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/mq/client/sub_client"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/protobuf/proto"
+	"strings"
+	"time"
+)
+
+var (
+	namespace   = flag.String("ns", "test", "namespace")
+	t           = flag.String("topic", "test", "topic")
+	seedBrokers = flag.String("brokers", "localhost:17777", "seed brokers")
+
+	clientId = flag.Uint("client_id", uint(util.RandomInt32()), "client id")
+)
+
+type MyRecord struct {
+	Key    []byte
+	Field1 []byte
+	Field2 string
+	Field3 int32
+	Field4 int64
+	Field5 float32
+	Field6 float64
+	Field7 bool
+}
+
+func FromSchemaRecordValue(recordValue *schema_pb.RecordValue) *MyRecord {
+	return &MyRecord{
+		Key:    recordValue.Fields["key"].GetBytesValue(),
+		Field1: recordValue.Fields["field1"].GetBytesValue(),
+		Field2: recordValue.Fields["field2"].GetStringValue(),
+		Field3: recordValue.Fields["field3"].GetInt32Value(),
+		Field4: recordValue.Fields["field4"].GetInt64Value(),
+		Field5: recordValue.Fields["field5"].GetFloatValue(),
+		Field6: recordValue.Fields["field6"].GetDoubleValue(),
+		Field7: recordValue.Fields["field7"].GetBoolValue(),
+	}
+}
+
+func main() {
+	flag.Parse()
+
+	subscriberConfig := &sub_client.SubscriberConfiguration{
+		ClientId:                fmt.Sprintf("client-%d", *clientId),
+		ConsumerGroup:           "test",
+		ConsumerGroupInstanceId: fmt.Sprintf("client-%d", *clientId),
+		GrpcDialOption:          grpc.WithTransportCredentials(insecure.NewCredentials()),
+	}
+
+	contentConfig := &sub_client.ContentConfiguration{
+		Topic:     topic.NewTopic(*namespace, *t),
+		Filter:    "",
+		StartTime: time.Unix(1, 1),
+	}
+
+	processorConfig := sub_client.ProcessorConfiguration{
+		ConcurrentPartitionLimit: 3,
+	}
+
+	brokers := strings.Split(*seedBrokers, ",")
+	subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, processorConfig)
+
+	counter := 0
+	subscriber.SetEachMessageFunc(func(key, value []byte) (bool, error) {
+		counter++
+		record := &schema_pb.RecordValue{}
+		proto.Unmarshal(value, record)
+		fmt.Printf("record: %v\n", record)
+		return true, nil
+	})
+
+	subscriber.SetCompletionFunc(func() {
+		glog.V(0).Infof("done received %d messages", counter)
+	})
+
+	if err := subscriber.Subscribe(); err != nil {
+		fmt.Println(err)
+	}
+
+}

weed/mq/client/pub_client/publish.go

@@ -2,12 +2,22 @@ package pub_client
 
 import (
 	"fmt"
+	"github.com/golang/protobuf/proto"
 	"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"time"
 )
 
 func (p *TopicPublisher) Publish(key, value []byte) error {
+	if p.config.RecordType != nil {
+		return fmt.Errorf("record type is set, use PublishRecord instead")
+	}
+	return p.doPublish(key, value)
+}
+
+func (p *TopicPublisher) doPublish(key, value []byte) error {
 	hashKey := util.HashToInt32(key) % pub_balancer.MaxPartitionCount
 	if hashKey < 0 {
 		hashKey = -hashKey
@@ -20,5 +30,31 @@ func (p *TopicPublisher) Publish(key, value []byte) error {
 	return inputBuffer.Enqueue(&mq_pb.DataMessage{
 		Key:   key,
 		Value: value,
+		TsNs:  time.Now().UnixNano(),
 	})
 }
+
+func (p *TopicPublisher) PublishRecord(key []byte, recordValue *schema_pb.RecordValue) error {
+	// serialize record value
+	value, err := proto.Marshal(recordValue)
+	if err != nil {
+		return fmt.Errorf("failed to marshal record value: %v", err)
+	}
+
+	return p.doPublish(key, value)
+}
+
+func (p *TopicPublisher) FinishPublish() error {
+	if inputBuffers, found := p.partition2Buffer.AllIntersections(0, pub_balancer.MaxPartitionCount); found {
+		for _, inputBuffer := range inputBuffers {
+			inputBuffer.Enqueue(&mq_pb.DataMessage{
+				TsNs:  time.Now().UnixNano(),
+				Ctrl: &mq_pb.ControlMessage{
+					IsClose: true,
+				},
+			})
+		}
+	}
+
+	return nil
+}

weed/mq/client/pub_client/publisher.go

@@ -5,6 +5,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util/buffered_queue"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
@@ -13,10 +14,11 @@ import (
 )
 
 type PublisherConfiguration struct {
-	Topic                     topic.Topic
-	CreateTopic               bool
-	CreateTopicPartitionCount int32
-	Brokers                   []string
+	Topic          topic.Topic
+	PartitionCount int32
+	Brokers        []string
+	PublisherName  string // for debugging
+	RecordType     *schema_pb.RecordType
 }
 
 type PublishClient struct {

weed/mq/client/pub_client/scheduler.go

@@ -12,6 +12,7 @@ import (
 	"log"
 	"sort"
 	"sync"
+	"sync/atomic"
 	"time"
 )
 
@@ -145,11 +146,13 @@ func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) erro
 				Partition:       job.Partition,
 				AckInterval:     128,
 				FollowerBrokers: job.FollowerBrokers,
+				PublisherName:   p.config.PublisherName,
 			},
 		},
 	}); err != nil {
 		return fmt.Errorf("send init message: %v", err)
 	}
+	// process the hello message
 	resp, err := stream.Recv()
 	if err != nil {
 		return fmt.Errorf("recv init response: %v", err)
@@ -158,31 +161,44 @@ func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) erro
 		return fmt.Errorf("init response error: %v", resp.Error)
 	}
 
+	var publishedTsNs int64
+	hasMoreData := int32(1)
+	var wg sync.WaitGroup
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
 		for {
 			ackResp, err := publishClient.Recv()
 			if err != nil {
 				e, _ := status.FromError(err)
 				if e.Code() == codes.Unknown && e.Message() == "EOF" {
+					log.Printf("publish to %s EOF", publishClient.Broker)
 					return
 				}
 				publishClient.Err = err
-				fmt.Printf("publish1 to %s error: %v\n", publishClient.Broker, err)
+				log.Printf("publish1 to %s error: %v\n", publishClient.Broker, err)
 				return
 			}
 			if ackResp.Error != "" {
 				publishClient.Err = fmt.Errorf("ack error: %v", ackResp.Error)
-				fmt.Printf("publish2 to %s error: %v\n", publishClient.Broker, ackResp.Error)
+				log.Printf("publish2 to %s error: %v\n", publishClient.Broker, ackResp.Error)
 				return
 			}
 			if ackResp.AckSequence > 0 {
-				log.Printf("ack %d", ackResp.AckSequence)
+				log.Printf("ack %d published %d hasMoreData:%d", ackResp.AckSequence, atomic.LoadInt64(&publishedTsNs), atomic.LoadInt32(&hasMoreData))
+			}
+			if atomic.LoadInt64(&publishedTsNs) <= ackResp.AckSequence && atomic.LoadInt32(&hasMoreData) == 0 {
+				return
 			}
 		}
 	}()
 
 	publishCounter := 0
 	for data, hasData := job.inputQueue.Dequeue(); hasData; data, hasData = job.inputQueue.Dequeue() {
+		if data.Ctrl != nil && data.Ctrl.IsClose {
+			// need to set this before sending to brokers, to avoid timing issue
+			atomic.StoreInt32(&hasMoreData, 0)
+		}
 		if err := publishClient.Send(&mq_pb.PublishMessageRequest{
 			Message: &mq_pb.PublishMessageRequest_Data{
 				Data: data,
@@ -191,14 +207,17 @@ func (p *TopicPublisher) doPublishToPartition(job *EachPartitionPublishJob) erro
 			return fmt.Errorf("send publish data: %v", err)
 		}
 		publishCounter++
+		atomic.StoreInt64(&publishedTsNs, data.TsNs)
 	}
-
-	if err := publishClient.CloseSend(); err != nil {
-		return fmt.Errorf("close send: %v", err)
+	if publishCounter > 0 {
+		wg.Wait()
+	} else {
+		// CloseSend would cancel the context on the server side
+		if err := publishClient.CloseSend(); err != nil {
+			return fmt.Errorf("close send: %v", err)
+		}
 	}
 
-	time.Sleep(3 * time.Second)
-
 	log.Printf("published %d messages to %v for topic partition %+v", publishCounter, job.LeaderBroker, job.Partition)
 
 	return nil
@@ -216,11 +235,13 @@ func (p *TopicPublisher) doConfigureTopic() (err error) {
 			func(client mq_pb.SeaweedMessagingClient) error {
 				_, err := client.ConfigureTopic(context.Background(), &mq_pb.ConfigureTopicRequest{
 					Topic:          p.config.Topic.ToPbTopic(),
-					PartitionCount: p.config.CreateTopicPartitionCount,
+					PartitionCount: p.config.PartitionCount,
+					RecordType:     p.config.RecordType, // TODO schema upgrade
 				})
 				return err
 			})
 		if err == nil {
+			lastErr = nil
 			return nil
 		} else {
 			lastErr = fmt.Errorf("%s: %v", brokerAddress, err)

weed/mq/client/sub_client/connect_to_sub_coordinator.go

@@ -91,26 +91,26 @@ func (sub *TopicSubscriber) onEachAssignment(assignment *mq_pb.SubscriberToSubCo
 	var wg sync.WaitGroup
 	semaphore := make(chan struct{}, sub.ProcessorConfig.ConcurrentPartitionLimit)
 
-	for _, assigned := range assignment.AssignedPartitions {
+	for _, assigned := range assignment.PartitionAssignments {
 		wg.Add(1)
 		semaphore <- struct{}{}
-		go func(partition *mq_pb.Partition, broker string) {
+		go func(assigned *mq_pb.BrokerPartitionAssignment) {
 			defer wg.Done()
 			defer func() { <-semaphore }()
-			glog.V(0).Infof("subscriber %s/%s assigned partition %+v at %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, partition, broker)
-			err := sub.onEachPartition(partition, broker)
+			glog.V(0).Infof("subscriber %s/%s assigned partition %+v at %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, assigned.Partition, assigned.LeaderBroker)
+			err := sub.onEachPartition(assigned)
 			if err != nil {
-				glog.V(0).Infof("subscriber %s/%s partition %+v at %v: %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, partition, broker, err)
+				glog.V(0).Infof("subscriber %s/%s partition %+v at %v: %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, assigned.Partition, assigned.LeaderBroker, err)
 			}
-		}(assigned.Partition, assigned.Broker)
+		}(assigned)
 	}
 
 	wg.Wait()
 }
 
-func (sub *TopicSubscriber) onEachPartition(partition *mq_pb.Partition, broker string) error {
+func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssignment) error {
 	// connect to the partition broker
-	return pb.WithBrokerGrpcClient(true, broker, sub.SubscriberConfig.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
+	return pb.WithBrokerGrpcClient(true, assigned.LeaderBroker, sub.SubscriberConfig.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
 		subscribeClient, err := client.SubscribeMessage(context.Background(), &mq_pb.SubscribeMessageRequest{
 			Message: &mq_pb.SubscribeMessageRequest_Init{
 				Init: &mq_pb.SubscribeMessageRequest_InitMessage{
@@ -118,11 +118,12 @@ func (sub *TopicSubscriber) onEachPartition(partition *mq_pb.Partition, broker s
 					ConsumerId:    sub.SubscriberConfig.ConsumerGroupInstanceId,
 					Topic:         sub.ContentConfig.Topic.ToPbTopic(),
 					PartitionOffset: &mq_pb.PartitionOffset{
-						Partition: partition,
+						Partition: assigned.Partition,
 						StartTsNs: sub.alreadyProcessedTsNs,
 						StartType: mq_pb.PartitionOffsetStartType_EARLIEST_IN_MEMORY,
 					},
 					Filter: sub.ContentConfig.Filter,
+					FollowerBrokers: assigned.FollowerBrokers,
 				},
 			},
 		})
@@ -131,7 +132,7 @@ func (sub *TopicSubscriber) onEachPartition(partition *mq_pb.Partition, broker s
 			return fmt.Errorf("create subscribe client: %v", err)
 		}
 
-		glog.V(0).Infof("subscriber %s/%s connected to partition %+v at %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, partition, broker)
+		glog.V(0).Infof("subscriber %s/%s connected to partition %+v at %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, assigned.Partition, assigned.LeaderBroker)
 
 		if sub.OnCompletionFunc != nil {
 			defer sub.OnCompletionFunc()

weed/mq/pub_balancer/allocate_test.go

@@ -12,7 +12,6 @@ func Test_allocateOneBroker(t *testing.T) {
 	brokers := cmap.New[*BrokerStats]()
 	brokers.SetIfAbsent("localhost:17777", &BrokerStats{
 		TopicPartitionCount: 0,
-		ConsumerCount:       0,
 		CpuUsagePercent:     0,
 	})
 
@@ -61,7 +60,6 @@ func testThem(t *testing.T, tests []struct {
 				assert.Equal(t, tt.wantAssignments[i].Partition.RangeStart, gotAssignment.Partition.RangeStart)
 				assert.Equal(t, tt.wantAssignments[i].Partition.RangeStop, gotAssignment.Partition.RangeStop)
 				assert.Equal(t, tt.wantAssignments[i].Partition.RingSize, gotAssignment.Partition.RingSize)
-				assert.Equal(t, tt.wantAssignments[i].Partition.UnixTimeNs, gotAssignment.Partition.UnixTimeNs)
 			}
 		})
 	}

weed/mq/pub_balancer/balance_brokers_test.go

@@ -12,7 +12,6 @@ func TestBalanceTopicPartitionOnBrokers(t *testing.T) {
 	brokers := cmap.New[*BrokerStats]()
 	broker1Stats := &BrokerStats{
 		TopicPartitionCount: 1,
-		ConsumerCount:       1,
 		CpuUsagePercent:     1,
 		TopicPartitionStats: cmap.New[*TopicPartitionStats](),
 	}
@@ -21,12 +20,9 @@ func TestBalanceTopicPartitionOnBrokers(t *testing.T) {
 			Topic:     topic.Topic{Namespace: "topic1", Name: "topic1"},
 			Partition: topic.Partition{RangeStart: 0, RangeStop: 512, RingSize: 1024},
 		},
-		ConsumerCount: 1,
-		IsLeader:      true,
 	})
 	broker2Stats := &BrokerStats{
 		TopicPartitionCount: 2,
-		ConsumerCount:       1,
 		CpuUsagePercent:     1,
 		TopicPartitionStats: cmap.New[*TopicPartitionStats](),
 	}
@@ -35,16 +31,12 @@ func TestBalanceTopicPartitionOnBrokers(t *testing.T) {
 			Topic:     topic.Topic{Namespace: "topic1", Name: "topic1"},
 			Partition: topic.Partition{RangeStart: 512, RangeStop: 1024, RingSize: 1024},
 		},
-		ConsumerCount: 1,
-		IsLeader:      true,
 	})
 	broker2Stats.TopicPartitionStats.Set("topic2:0", &TopicPartitionStats{
 		TopicPartition: topic.TopicPartition{
 			Topic:     topic.Topic{Namespace: "topic2", Name: "topic2"},
 			Partition: topic.Partition{RangeStart: 0, RangeStop: 1024, RingSize: 1024},
 		},
-		ConsumerCount: 1,
-		IsLeader:      true,
 	})
 	brokers.Set("broker1", broker1Stats)
 	brokers.Set("broker2", broker2Stats)

weed/mq/pub_balancer/broker_stats.go

@@ -9,15 +9,16 @@ import (
 
 type BrokerStats struct {
 	TopicPartitionCount int32
-	ConsumerCount       int32
+	PublisherCount      int32
+	SubscriberCount     int32
 	CpuUsagePercent     int32
 	TopicPartitionStats cmap.ConcurrentMap[string, *TopicPartitionStats] // key: topic_partition
 	Topics              []topic.Topic
 }
 type TopicPartitionStats struct {
 	topic.TopicPartition
-	ConsumerCount int32
-	IsLeader      bool
+	PublisherCount int32
+	SubscriberCount int32
 }
 
 func NewBrokerStats() *BrokerStats {
@@ -26,15 +27,15 @@ func NewBrokerStats() *BrokerStats {
 	}
 }
 func (bs *BrokerStats) String() string {
-	return fmt.Sprintf("BrokerStats{TopicPartitionCount:%d, ConsumerCount:%d, CpuUsagePercent:%d, Stats:%+v}",
-		bs.TopicPartitionCount, bs.ConsumerCount, bs.CpuUsagePercent, bs.TopicPartitionStats.Items())
+	return fmt.Sprintf("BrokerStats{TopicPartitionCount:%d, Publishers:%d, Subscribers:%d CpuUsagePercent:%d, Stats:%+v}",
+		bs.TopicPartitionCount, bs.PublisherCount, bs.SubscriberCount, bs.CpuUsagePercent, bs.TopicPartitionStats.Items())
 }
 
 func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) {
 	bs.TopicPartitionCount = int32(len(stats.Stats))
 	bs.CpuUsagePercent = stats.CpuUsagePercent
 
-	var consumerCount int32
+	var publisherCount, subscriberCount int32
 	currentTopicPartitions := bs.TopicPartitionStats.Items()
 	for _, topicPartitionStats := range stats.Stats {
 		tps := &TopicPartitionStats{
@@ -47,10 +48,11 @@ func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) {
 					UnixTimeNs: topicPartitionStats.Partition.UnixTimeNs,
 				},
 			},
-			ConsumerCount: topicPartitionStats.ConsumerCount,
-			IsLeader:      topicPartitionStats.IsLeader,
+			PublisherCount: topicPartitionStats.PublisherCount,
+			SubscriberCount: topicPartitionStats.SubscriberCount,
 		}
-		consumerCount += topicPartitionStats.ConsumerCount
+		publisherCount += topicPartitionStats.PublisherCount
+		subscriberCount += topicPartitionStats.SubscriberCount
 		key := tps.TopicPartition.String()
 		bs.TopicPartitionStats.Set(key, tps)
 		delete(currentTopicPartitions, key)
@@ -59,8 +61,8 @@ func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) {
 	for key := range currentTopicPartitions {
 		bs.TopicPartitionStats.Remove(key)
 	}
-	bs.ConsumerCount = consumerCount
-
+	bs.PublisherCount = publisherCount
+	bs.SubscriberCount = subscriberCount
 }
 
 func (bs *BrokerStats) RegisterAssignment(t *mq_pb.Topic, partition *mq_pb.Partition, isAdd bool) {
@@ -74,8 +76,8 @@ func (bs *BrokerStats) RegisterAssignment(t *mq_pb.Topic, partition *mq_pb.Parti
 				UnixTimeNs: partition.UnixTimeNs,
 			},
 		},
-		ConsumerCount: 0,
-		IsLeader:      true,
+		PublisherCount: 0,
+		SubscriberCount: 0,
 	}
 	key := tps.TopicPartition.String()
 	if isAdd {

weed/mq/pub_balancer/repair.go

@@ -14,8 +14,7 @@ func (balancer *Balancer) RepairTopics() []BalanceAction {
 }
 
 type TopicPartitionInfo struct {
-	Leader    string
-	Followers []string
+	Broker string
 }
 
 // RepairMissingTopicPartitions check the stats of all brokers,
@@ -38,11 +37,7 @@ func RepairMissingTopicPartitions(brokers cmap.ConcurrentMap[string, *BrokerStat
 				tpi = &TopicPartitionInfo{}
 				topicPartitionToInfo[topicPartitionStat.Partition] = tpi
 			}
-			if topicPartitionStat.IsLeader {
-				tpi.Leader = broker
-			} else {
-				tpi.Followers = append(tpi.Followers, broker)
-			}
+			tpi.Broker = broker
 		}
 	}
 

weed/mq/schema/schema.go

@@ -0,0 +1,26 @@
+package schema
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+type Schema struct {
+	RecordType *schema_pb.RecordType
+	fieldMap map[string]*schema_pb.Field
+}
+
+func NewSchema(recordType *schema_pb.RecordType) (*Schema, error) {
+	fieldMap := make( map[string]*schema_pb.Field)
+	for _, field := range recordType.Fields {
+		fieldMap[field.Name] = field
+	}
+	return &Schema{
+		RecordType: recordType,
+		fieldMap: fieldMap,
+	}, nil
+}
+
+func (s *Schema) GetField(name string) (*schema_pb.Field, bool) {
+	field, ok := s.fieldMap[name]
+	return field, ok
+}

weed/mq/schema/schema_builder.go

@@ -0,0 +1,52 @@
+package schema
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"sort"
+)
+
+var (
+	TypeBoolean = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_BOOL}}
+	TypeInt32   = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_INT32}}
+	TypeInt64  = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_INT64}}
+	TypeFloat  = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_FLOAT}}
+	TypeDouble = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_DOUBLE}}
+	TypeBytes   = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_BYTES}}
+	TypeString  = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_STRING}}
+)
+
+type RecordTypeBuilder struct {
+	recordType *schema_pb.RecordType
+}
+
+func RecordTypeBegin() *RecordTypeBuilder {
+	return &RecordTypeBuilder{recordType: &schema_pb.RecordType{}}
+}
+
+func (rtb *RecordTypeBuilder) RecordTypeEnd() *schema_pb.RecordType {
+	// be consistent with parquet.node.go `func (g Group) Fields() []Field`
+	sort.Slice(rtb.recordType.Fields, func(i, j int) bool {
+		return rtb.recordType.Fields[i].Name < rtb.recordType.Fields[j].Name
+	})
+	return rtb.recordType
+}
+
+func (rtb *RecordTypeBuilder) WithField(name string, scalarType *schema_pb.Type) *RecordTypeBuilder {
+	rtb.recordType.Fields = append(rtb.recordType.Fields, &schema_pb.Field{
+		Name: name,
+		Type: scalarType,
+	})
+	return rtb
+}
+
+func (rtb *RecordTypeBuilder) WithRecordField(name string, recordType *schema_pb.RecordType) *RecordTypeBuilder {
+	rtb.recordType.Fields = append(rtb.recordType.Fields, &schema_pb.Field{
+		Name: name,
+		Type: &schema_pb.Type{Kind: &schema_pb.Type_RecordType{RecordType: recordType}},
+	})
+	return rtb
+}
+
+func ListOf(elementType *schema_pb.Type) *schema_pb.Type {
+	return &schema_pb.Type{Kind: &schema_pb.Type_ListType{ListType: &schema_pb.ListType{ElementType: elementType}}}
+}

weed/mq/schema/schema_test.go

@@ -0,0 +1,85 @@
+package schema
+
+import (
+	"encoding/json"
+	"github.com/golang/protobuf/proto"
+	. "github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestEnumScalarType(t *testing.T) {
+	tests := []struct {
+		name     string
+		enum     ScalarType
+		expected int32
+	}{
+		{"Boolean", ScalarType_BOOL, 0},
+		{"Integer", ScalarType_INT32, 1},
+		{"Long", ScalarType_INT64, 3},
+		{"Float", ScalarType_FLOAT, 4},
+		{"Double", ScalarType_DOUBLE, 5},
+		{"Bytes", ScalarType_BYTES, 6},
+		{"String", ScalarType_STRING, 7},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.expected, int32(tt.enum))
+		})
+	}
+}
+
+func TestField(t *testing.T) {
+	field := &Field{
+		Name:        "field_name",
+		Type:        &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}},
+		FieldIndex:       1,
+		IsRepeated:  false,
+	}
+	assert.NotNil(t, field)
+}
+
+func TestRecordType(t *testing.T) {
+	subRecord := &RecordType{
+		Fields: []*Field{
+			{
+				Name:        "field_1",
+				Type:        &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}},
+				FieldIndex:       1,
+				IsRepeated:  false,
+			},
+			{
+				Name:        "field_2",
+				Type:        &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_STRING}},
+				FieldIndex:       2,
+				IsRepeated:  false,
+			},
+		},
+	}
+	record := &RecordType{
+		Fields: []*Field{
+			{
+				Name:        "field_key",
+				Type:        &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}},
+				FieldIndex:       1,
+				IsRepeated:  false,
+			},
+			{
+				Name:        "field_record",
+				Type:        &Type{Kind: &Type_RecordType{RecordType: subRecord}},
+				FieldIndex:       2,
+				IsRepeated:  false,
+			},
+		},
+	}
+
+	// serialize record to protobuf text marshalling
+	text := proto.MarshalTextString(record)
+	println(text)
+
+	bytes, _ := json.Marshal(record)
+	println(string(bytes))
+
+	assert.NotNil(t, record)
+}

weed/mq/schema/struct_to_schema.go

@@ -0,0 +1,68 @@
+package schema
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"reflect"
+)
+
+func StructToSchema(instance any) *schema_pb.RecordType {
+	myType := reflect.TypeOf(instance)
+	if myType.Kind() != reflect.Struct {
+		return nil
+	}
+	st := reflectTypeToSchemaType(myType)
+	return st.GetRecordType()
+}
+
+func reflectTypeToSchemaType(t reflect.Type) *schema_pb.Type {
+	switch t.Kind() {
+	case reflect.Bool:
+		return TypeBoolean
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32:
+		return TypeInt32
+	case reflect.Int64:
+		return TypeInt64
+	case reflect.Float32:
+		return TypeFloat
+	case reflect.Float64:
+		return TypeDouble
+	case reflect.String:
+		return TypeString
+	case reflect.Slice:
+		switch t.Elem().Kind() {
+		case reflect.Uint8:
+			return TypeBytes
+		default:
+			if st := reflectTypeToSchemaType(t.Elem()); st != nil {
+				return &schema_pb.Type{
+					Kind: &schema_pb.Type_ListType{
+						ListType: &schema_pb.ListType{
+							ElementType: st,
+						},
+					},
+				}
+			}
+		}
+	case reflect.Struct:
+		recordType := &schema_pb.RecordType{}
+		for i := 0; i < t.NumField(); i++ {
+			field := t.Field(i)
+			fieldType := field.Type
+			fieldName := field.Name
+			schemaField := reflectTypeToSchemaType(fieldType)
+			if schemaField == nil {
+				return nil
+			}
+			recordType.Fields = append(recordType.Fields, &schema_pb.Field{
+				Name: fieldName,
+				Type: schemaField,
+			})
+		}
+		return &schema_pb.Type{
+			Kind: &schema_pb.Type_RecordType{
+				RecordType: recordType,
+			},
+		}
+	}
+	return nil
+}

weed/mq/schema/struct_to_schema_test.go

@@ -0,0 +1,119 @@
+package schema
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestStructToSchema(t *testing.T) {
+	type args struct {
+		instance any
+	}
+	tests := []struct {
+		name string
+		args args
+		want *schema_pb.RecordType
+	}{
+		{
+			name: "scalar type",
+			args: args{
+				instance: 1,
+			},
+			want: nil,
+		},
+		{
+			name: "simple struct type",
+			args: args{
+				instance: struct {
+					Field1 int
+					Field2 string
+				}{},
+			},
+			want: RecordTypeBegin().
+				WithField("Field1", TypeInt32).
+				WithField("Field2", TypeString).
+				RecordTypeEnd(),
+		},
+		{
+			name: "simple list",
+			args: args{
+				instance: struct {
+					Field1 []int
+					Field2 string
+				}{},
+			},
+			want: RecordTypeBegin().
+				WithField("Field1", ListOf(TypeInt32)).
+				WithField("Field2", TypeString).
+				RecordTypeEnd(),
+		},
+		{
+			name: "simple []byte",
+			args: args{
+				instance: struct {
+					Field2 []byte
+				}{},
+			},
+			want: RecordTypeBegin().
+				WithField("Field2", TypeBytes).
+				RecordTypeEnd(),
+		},
+		{
+			name: "nested simpe structs",
+			args: args{
+				instance: struct {
+					Field1 int
+					Field2 struct {
+						Field3 string
+						Field4 int
+					}
+				}{},
+			},
+			want: RecordTypeBegin().
+				WithField("Field1", TypeInt32).
+				WithRecordField("Field2",
+					RecordTypeBegin().
+						WithField("Field3", TypeString).
+						WithField("Field4", TypeInt32).
+					RecordTypeEnd(),
+				).
+				RecordTypeEnd(),
+		},
+		{
+			name: "nested struct type",
+			args: args{
+				instance: struct {
+					Field1 int
+					Field2 struct {
+						Field3 string
+						Field4 []int
+						Field5 struct {
+							Field6 string
+							Field7 []byte
+						}
+					}
+				}{},
+			},
+			want: RecordTypeBegin().
+				WithField("Field1", TypeInt32).
+				WithRecordField("Field2", RecordTypeBegin().
+					WithField("Field3", TypeString).
+					WithField("Field4", ListOf(TypeInt32)).
+					WithRecordField("Field5",
+						RecordTypeBegin().
+							WithField("Field6", TypeString).
+							WithField("Field7", TypeBytes).
+						RecordTypeEnd(),
+					).RecordTypeEnd(),
+				).
+				RecordTypeEnd(),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equalf(t, tt.want, StructToSchema(tt.args.instance), "StructToSchema(%v)", tt.args.instance)
+		})
+	}
+}

weed/mq/schema/to_parquet_levels.go

@@ -0,0 +1,58 @@
+package schema
+
+import (
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+type ParquetLevels struct {
+	startColumnIndex int
+	endColumnIndex int
+	definitionDepth int
+	levels map[string]*ParquetLevels
+}
+
+func ToParquetLevels(recordType *schema_pb.RecordType) (*ParquetLevels, error) {
+	return toRecordTypeLevels(recordType, 0, 0)
+}
+
+func toFieldTypeLevels(fieldType *schema_pb.Type, startColumnIndex, definitionDepth int) (*ParquetLevels, error) {
+	switch fieldType.Kind.(type) {
+	case *schema_pb.Type_ScalarType:
+		return toFieldTypeScalarLevels(fieldType.GetScalarType(),  startColumnIndex, definitionDepth)
+	case *schema_pb.Type_RecordType:
+		return toRecordTypeLevels(fieldType.GetRecordType(), startColumnIndex, definitionDepth)
+	case *schema_pb.Type_ListType:
+		return toFieldTypeListLevels(fieldType.GetListType(), startColumnIndex, definitionDepth)
+	}
+	return nil, fmt.Errorf("unknown field type: %T", fieldType.Kind)
+}
+
+func toFieldTypeListLevels(listType *schema_pb.ListType, startColumnIndex, definitionDepth int) (*ParquetLevels, error) {
+	return toFieldTypeLevels(listType.ElementType, startColumnIndex, definitionDepth)
+}
+
+func toFieldTypeScalarLevels(scalarType schema_pb.ScalarType, startColumnIndex, definitionDepth int) (*ParquetLevels, error) {
+	return &ParquetLevels{
+		startColumnIndex: startColumnIndex,
+		endColumnIndex: startColumnIndex + 1,
+		definitionDepth: definitionDepth,
+	}, nil
+}
+func toRecordTypeLevels(recordType *schema_pb.RecordType, startColumnIndex, definitionDepth int) (*ParquetLevels, error) {
+	recordTypeLevels := &ParquetLevels{
+		startColumnIndex: startColumnIndex,
+		definitionDepth: definitionDepth,
+		levels: make(map[string]*ParquetLevels),
+	}
+	for _, field := range recordType.Fields {
+		fieldTypeLevels, err := toFieldTypeLevels(field.Type, startColumnIndex, definitionDepth+1)
+		if err != nil {
+			return nil, err
+		}
+		recordTypeLevels.levels[field.Name] = fieldTypeLevels
+		startColumnIndex = fieldTypeLevels.endColumnIndex
+	}
+	recordTypeLevels.endColumnIndex = startColumnIndex
+	return recordTypeLevels, nil
+}

weed/mq/schema/to_parquet_levels_test.go

@@ -0,0 +1,102 @@
+package schema
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestToParquetLevels(t *testing.T) {
+	type args struct {
+		recordType *schema_pb.RecordType
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *ParquetLevels
+	}{
+		{
+			name: "nested type",
+			args: args{
+				RecordTypeBegin().
+					WithField("ID", TypeInt64).
+					WithField("CreatedAt", TypeInt64).
+					WithRecordField("Person",
+						RecordTypeBegin().
+							WithField("zName", TypeString).
+							WithField("emails", ListOf(TypeString)).
+						RecordTypeEnd()).
+					WithField("Company", TypeString).
+					WithRecordField("Address",
+						RecordTypeBegin().
+							WithField("Street", TypeString).
+							WithField("City", TypeString).
+						RecordTypeEnd()).
+					RecordTypeEnd(),
+			},
+			want: &ParquetLevels{
+				startColumnIndex: 0,
+				endColumnIndex:   7,
+				definitionDepth:  0,
+				levels: map[string]*ParquetLevels{
+					"Address": {
+						startColumnIndex: 0,
+						endColumnIndex:   2,
+						definitionDepth:  1,
+						levels: map[string]*ParquetLevels{
+							"City": {
+								startColumnIndex: 0,
+								endColumnIndex:   1,
+								definitionDepth:  2,
+							},
+							"Street": {
+								startColumnIndex: 1,
+								endColumnIndex:   2,
+								definitionDepth:  2,
+							},
+						},
+					},
+					"Company": {
+						startColumnIndex: 2,
+						endColumnIndex:   3,
+						definitionDepth:  1,
+					},
+					"CreatedAt": {
+						startColumnIndex: 3,
+						endColumnIndex:   4,
+						definitionDepth:  1,
+					},
+					"ID": {
+						startColumnIndex: 4,
+						endColumnIndex:   5,
+						definitionDepth:  1,
+					},
+					"Person": {
+						startColumnIndex: 5,
+						endColumnIndex:   7,
+						definitionDepth:  1,
+						levels: map[string]*ParquetLevels{
+							"emails": {
+								startColumnIndex: 5,
+								endColumnIndex:   6,
+								definitionDepth:  2,
+							},
+							"zName": {
+								startColumnIndex: 6,
+								endColumnIndex:   7,
+								definitionDepth:  2,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ToParquetLevels(tt.args.recordType)
+			assert.Nil(t, err)
+			assert.Equalf(t, tt.want, got, "ToParquetLevels(%v)", tt.args.recordType)
+		})
+	}
+}

weed/mq/schema/to_parquet_schema.go

@@ -0,0 +1,76 @@
+package schema
+
+import (
+	"fmt"
+	parquet "github.com/parquet-go/parquet-go"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+func ToParquetSchema(topicName string, recordType *schema_pb.RecordType) (*parquet.Schema, error) {
+	rootNode, err := toParquetFieldTypeRecord(recordType)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert record type to parquet schema: %v", err)
+	}
+
+	// Fields are sorted by name, so the value should be sorted also
+	// the sorting is inside parquet.`func (g Group) Fields() []Field`
+	return parquet.NewSchema(topicName, rootNode), nil
+}
+
+func toParquetFieldType(fieldType *schema_pb.Type) (dataType parquet.Node, err error) {
+	switch fieldType.Kind.(type) {
+	case *schema_pb.Type_ScalarType:
+		dataType, err = toParquetFieldTypeScalar(fieldType.GetScalarType())
+		dataType = parquet.Optional(dataType)
+	case *schema_pb.Type_RecordType:
+		dataType, err = toParquetFieldTypeRecord(fieldType.GetRecordType())
+		dataType = parquet.Optional(dataType)
+	case *schema_pb.Type_ListType:
+		dataType, err = toParquetFieldTypeList(fieldType.GetListType())
+	default:
+		return nil, fmt.Errorf("unknown field type: %T", fieldType.Kind)
+	}
+
+
+	return dataType, err
+}
+
+func toParquetFieldTypeList(listType *schema_pb.ListType) (parquet.Node, error) {
+	elementType, err := toParquetFieldType(listType.ElementType)
+	if err != nil {
+		return nil, err
+	}
+	return parquet.Repeated(elementType), nil
+}
+
+func toParquetFieldTypeScalar(scalarType schema_pb.ScalarType) (parquet.Node, error) {
+	switch scalarType {
+	case schema_pb.ScalarType_BOOL:
+		return parquet.Leaf(parquet.BooleanType), nil
+	case schema_pb.ScalarType_INT32:
+		return parquet.Leaf(parquet.Int32Type), nil
+	case schema_pb.ScalarType_INT64:
+		return parquet.Leaf(parquet.Int64Type), nil
+	case schema_pb.ScalarType_FLOAT:
+		return parquet.Leaf(parquet.FloatType), nil
+	case schema_pb.ScalarType_DOUBLE:
+		return parquet.Leaf(parquet.DoubleType), nil
+	case schema_pb.ScalarType_BYTES:
+		return parquet.Leaf(parquet.ByteArrayType), nil
+	case schema_pb.ScalarType_STRING:
+		return parquet.Leaf(parquet.ByteArrayType), nil
+	default:
+		return nil, fmt.Errorf("unknown scalar type: %v", scalarType)
+	}
+}
+func toParquetFieldTypeRecord(recordType *schema_pb.RecordType) (parquet.Node, error) {
+	recordNode := parquet.Group{}
+	for _, field := range recordType.Fields {
+		parquetFieldType, err := toParquetFieldType(field.Type)
+		if err != nil {
+			return nil, err
+		}
+		recordNode[field.Name] = parquetFieldType
+	}
+	return recordNode, nil
+}

weed/mq/schema/to_parquet_value.go

@@ -0,0 +1,92 @@
+package schema
+
+import (
+	"fmt"
+	parquet "github.com/parquet-go/parquet-go"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+func rowBuilderVisit(rowBuilder *parquet.RowBuilder, fieldType *schema_pb.Type, levels *ParquetLevels, fieldValue *schema_pb.Value) (err error) {
+	switch fieldType.Kind.(type) {
+	case *schema_pb.Type_ScalarType:
+		var parquetValue parquet.Value
+		parquetValue, err = toParquetValue(fieldValue)
+		if err != nil {
+			return
+		}
+		rowBuilder.Add(levels.startColumnIndex, parquetValue)
+		// fmt.Printf("rowBuilder.Add %d %v\n", columnIndex, parquetValue)
+	case *schema_pb.Type_ListType:
+		rowBuilder.Next(levels.startColumnIndex)
+		// fmt.Printf("rowBuilder.Next %d\n", columnIndex)
+
+		elementType := fieldType.GetListType().ElementType
+		for _, value := range fieldValue.GetListValue().Values {
+			if err = rowBuilderVisit(rowBuilder, elementType, levels, value); err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+func AddRecordValue(rowBuilder *parquet.RowBuilder, recordType *schema_pb.RecordType, parquetLevels *ParquetLevels, recordValue *schema_pb.RecordValue) error {
+	visitor := func(fieldType *schema_pb.Type, levels *ParquetLevels, fieldValue *schema_pb.Value) (err error) {
+		return rowBuilderVisit(rowBuilder, fieldType, levels, fieldValue)
+	}
+	fieldType := &schema_pb.Type{Kind: &schema_pb.Type_RecordType{RecordType: recordType}}
+	fieldValue := &schema_pb.Value{Kind: &schema_pb.Value_RecordValue{RecordValue: recordValue}}
+	return doVisitValue(fieldType, parquetLevels, fieldValue, visitor)
+}
+
+// typeValueVisitor is a function that is called for each value in a schema_pb.Value
+// Find the column index.
+// intended to be used in RowBuilder.Add(columnIndex, value)
+type typeValueVisitor func(fieldType *schema_pb.Type, levels *ParquetLevels, fieldValue *schema_pb.Value) (err error)
+
+// endIndex is exclusive
+// same logic as RowBuilder.configure in row_builder.go
+func doVisitValue(fieldType *schema_pb.Type, levels *ParquetLevels, fieldValue *schema_pb.Value, visitor typeValueVisitor) (err error) {
+	switch fieldType.Kind.(type) {
+	case *schema_pb.Type_ScalarType:
+		return visitor(fieldType, levels, fieldValue)
+	case *schema_pb.Type_ListType:
+		return visitor(fieldType, levels, fieldValue)
+	case *schema_pb.Type_RecordType:
+		for _, field := range fieldType.GetRecordType().Fields {
+			fieldValue, found := fieldValue.GetRecordValue().Fields[field.Name]
+			if !found {
+				// TODO check this if no such field found
+				continue
+			}
+			fieldLevels := levels.levels[field.Name]
+			err = doVisitValue(field.Type, fieldLevels, fieldValue, visitor)
+			if err != nil {
+				return
+			}
+		}
+		return
+	}
+	return
+}
+
+func toParquetValue(value *schema_pb.Value) (parquet.Value,  error) {
+	switch value.Kind.(type) {
+	case *schema_pb.Value_BoolValue:
+		return parquet.BooleanValue(value.GetBoolValue()), nil
+	case *schema_pb.Value_Int32Value:
+		return parquet.Int32Value(value.GetInt32Value()), nil
+	case *schema_pb.Value_Int64Value:
+		return parquet.Int64Value(value.GetInt64Value()), nil
+	case *schema_pb.Value_FloatValue:
+		return parquet.FloatValue(value.GetFloatValue()), nil
+	case *schema_pb.Value_DoubleValue:
+		return parquet.DoubleValue(value.GetDoubleValue()), nil
+	case *schema_pb.Value_BytesValue:
+		return parquet.ByteArrayValue(value.GetBytesValue()), nil
+	case *schema_pb.Value_StringValue:
+		return parquet.ByteArrayValue([]byte(value.GetStringValue())), nil
+	default:
+		return parquet.NullValue(), fmt.Errorf("unknown value type: %T", value.Kind)
+	}
+}

weed/mq/schema/to_schema_value.go

@@ -0,0 +1,85 @@
+package schema
+
+import (
+	"fmt"
+	"github.com/parquet-go/parquet-go"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+// ToRecordValue converts a parquet.Row to a schema_pb.RecordValue
+// This does not work or did not test with nested structures.
+// Using this may fail to convert the parquet.Row to schema_pb.RecordValue
+func ToRecordValue(recordType *schema_pb.RecordType, parquetLevels *ParquetLevels, row parquet.Row) (*schema_pb.RecordValue, error) {
+	values := []parquet.Value(row)
+	recordValue, _, err := toRecordValue(recordType, parquetLevels, values, 0)
+	if err != nil {
+		return nil, err
+	}
+	return recordValue.GetRecordValue(), nil
+}
+
+func ToValue(t *schema_pb.Type, levels *ParquetLevels, values []parquet.Value, valueIndex int) (value *schema_pb.Value, endValueIndex int, err error) {
+	switch t.Kind.(type) {
+	case *schema_pb.Type_ScalarType:
+		return toScalarValue(t.GetScalarType(), levels, values, valueIndex)
+	case *schema_pb.Type_ListType:
+		return toListValue(t.GetListType(), levels, values, valueIndex)
+	case *schema_pb.Type_RecordType:
+		return toRecordValue(t.GetRecordType(), levels, values, valueIndex)
+	}
+	return nil, valueIndex, fmt.Errorf("unsupported type: %v", t)
+}
+
+func toRecordValue(recordType *schema_pb.RecordType, levels *ParquetLevels, values []parquet.Value, valueIndex int) (*schema_pb.Value, int, error) {
+	recordValue := schema_pb.RecordValue{Fields: make(map[string]*schema_pb.Value)}
+	for _, field := range recordType.Fields {
+		fieldLevels := levels.levels[field.Name]
+		fieldValue, endValueIndex, err := ToValue(field.Type, fieldLevels, values, valueIndex)
+		if err != nil {
+			return nil, 0, err
+		}
+		valueIndex = endValueIndex
+		recordValue.Fields[field.Name] = fieldValue
+	}
+	return &schema_pb.Value{Kind: &schema_pb.Value_RecordValue{RecordValue: &recordValue}}, valueIndex, nil
+}
+
+func toListValue(listType *schema_pb.ListType, levels *ParquetLevels, values []parquet.Value, valueIndex int) (listValue *schema_pb.Value, endValueIndex int, err error) {
+	listValues := make([]*schema_pb.Value, 0)
+	var value *schema_pb.Value
+	for ;valueIndex < len(values); {
+		if values[valueIndex].Column() != levels.startColumnIndex {
+			break
+		}
+		value, valueIndex, err = ToValue(listType.ElementType, levels, values, valueIndex)
+		if err != nil {
+			return nil, valueIndex, err
+		}
+		listValues = append(listValues, value)
+	}
+	return &schema_pb.Value{Kind: &schema_pb.Value_ListValue{ListValue: &schema_pb.ListValue{Values: listValues}}}, valueIndex, nil
+}
+
+func toScalarValue(scalarType schema_pb.ScalarType, levels *ParquetLevels, values []parquet.Value, valueIndex int) (*schema_pb.Value, int, error) {
+	value := values[valueIndex]
+	if value.Column() != levels.startColumnIndex {
+		return nil, valueIndex, nil
+	}
+	switch scalarType {
+	case schema_pb.ScalarType_BOOL:
+		return &schema_pb.Value{Kind: &schema_pb.Value_BoolValue{BoolValue: value.Boolean()}}, valueIndex+1, nil
+	case schema_pb.ScalarType_INT32:
+		return &schema_pb.Value{Kind: &schema_pb.Value_Int32Value{Int32Value: value.Int32()}}, valueIndex+1, nil
+	case schema_pb.ScalarType_INT64:
+		return &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: value.Int64()}}, valueIndex+1,  nil
+	case schema_pb.ScalarType_FLOAT:
+		return &schema_pb.Value{Kind: &schema_pb.Value_FloatValue{FloatValue: value.Float()}}, valueIndex+1, nil
+	case schema_pb.ScalarType_DOUBLE:
+		return &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: value.Double()}}, valueIndex+1, nil
+	case schema_pb.ScalarType_BYTES:
+		return &schema_pb.Value{Kind: &schema_pb.Value_BytesValue{BytesValue: value.ByteArray()}}, valueIndex+1, nil
+	case schema_pb.ScalarType_STRING:
+		return &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: string(value.ByteArray())}}, valueIndex+1, nil
+	}
+	return nil, valueIndex, fmt.Errorf("unsupported scalar type: %v", scalarType)
+}

weed/mq/schema/value_builder.go

@@ -0,0 +1,113 @@
+package schema
+
+import "github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+
+// RecordValueBuilder helps in constructing RecordValue protobuf messages
+type RecordValueBuilder struct {
+	recordValue *schema_pb.RecordValue
+}
+
+// RecordBegin creates a new RecordValueBuilder instance
+func RecordBegin() *RecordValueBuilder {
+	return &RecordValueBuilder{recordValue: &schema_pb.RecordValue{Fields: make(map[string]*schema_pb.Value)}}
+}
+
+// RecordEnd returns the constructed RecordValue message
+func (rvb *RecordValueBuilder) RecordEnd() *schema_pb.RecordValue {
+	return rvb.recordValue
+}
+
+func (rvb *RecordValueBuilder) SetBool(key string, value bool) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_BoolValue{BoolValue: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetInt32(key string, value int32) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_Int32Value{Int32Value: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetInt64(key string, value int64) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetFloat(key string, value float32) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_FloatValue{FloatValue: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetDouble(key string, value float64) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetBytes(key string, value []byte) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_BytesValue{BytesValue: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetString(key string, value string) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: value}}
+	return rvb
+}
+func (rvb *RecordValueBuilder) SetRecord(key string, value *schema_pb.RecordValue) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_RecordValue{RecordValue: value}}
+	return rvb
+}
+
+func (rvb *RecordValueBuilder) addListValue(key string, values []*schema_pb.Value) *RecordValueBuilder {
+	rvb.recordValue.Fields[key] = &schema_pb.Value{Kind: &schema_pb.Value_ListValue{ListValue: &schema_pb.ListValue{Values: values}}}
+	return rvb
+}
+
+func (rvb *RecordValueBuilder) SetBoolList(key string, values ...bool) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_BoolValue{BoolValue: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetInt32List(key string, values ...int32) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_Int32Value{Int32Value: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetInt64List(key string, values ...int64) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetFloatList(key string, values ...float32) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_FloatValue{FloatValue: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetDoubleList(key string, values ...float64) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetBytesList(key string, values ...[]byte) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_BytesValue{BytesValue: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetStringList(key string, values ...string) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}
+func (rvb *RecordValueBuilder) SetRecordList(key string, values ...*schema_pb.RecordValue) *RecordValueBuilder {
+	var listValues []*schema_pb.Value
+	for _, v := range values {
+		listValues = append(listValues, &schema_pb.Value{Kind: &schema_pb.Value_RecordValue{RecordValue: v}})
+	}
+	return rvb.addListValue(key, listValues)
+}

weed/mq/schema/write_parquet_test.go

@@ -0,0 +1,153 @@
+package schema
+
+import (
+	"fmt"
+	"github.com/parquet-go/parquet-go"
+	"github.com/parquet-go/parquet-go/compress/zstd"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+	"io"
+	"os"
+	"testing"
+)
+
+func TestWriteReadParquet(t *testing.T) {
+	// create a schema_pb.RecordType
+	recordType := RecordTypeBegin().
+		WithField("ID", TypeInt64).
+		WithField("CreatedAt", TypeInt64).
+		WithRecordField("Person",
+			RecordTypeBegin().
+				WithField("zName", TypeString).
+				WithField("emails", ListOf(TypeString)).
+			RecordTypeEnd()).
+		WithField("Company", TypeString).
+		WithRecordField("Address",
+			RecordTypeBegin().
+				WithField("Street", TypeString).
+				WithField("City", TypeString).
+			RecordTypeEnd()).
+		RecordTypeEnd()
+	fmt.Printf("RecordType: %v\n", recordType)
+
+	// create a parquet schema
+	parquetSchema, err := ToParquetSchema("example", recordType)
+	if err != nil {
+		t.Fatalf("ToParquetSchema failed: %v", err)
+	}
+	fmt.Printf("ParquetSchema: %v\n", parquetSchema)
+
+	fmt.Printf("Go Type: %+v\n", parquetSchema.GoType())
+
+	filename := "example.parquet"
+
+	count := 3
+
+	testWritingParquetFile(t, count, filename, parquetSchema, recordType)
+
+	total := testReadingParquetFile(t, filename, parquetSchema, recordType)
+
+	if total != count {
+		t.Fatalf("total != 128*1024: %v", total)
+	}
+
+	if err = os.Remove(filename); err != nil {
+		t.Fatalf("os.Remove failed: %v", err)
+	}
+
+}
+
+func testWritingParquetFile(t *testing.T, count int, filename string, parquetSchema *parquet.Schema, recordType *schema_pb.RecordType) {
+	parquetLevels, err := ToParquetLevels(recordType)
+	if err != nil {
+		t.Fatalf("ToParquetLevels failed: %v", err)
+	}
+
+	// create a parquet file
+	file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0664)
+	if err != nil {
+		t.Fatalf("os.Open failed: %v", err)
+	}
+	defer file.Close()
+	writer := parquet.NewWriter(file, parquetSchema, parquet.Compression(&zstd.Codec{Level: zstd.DefaultLevel}))
+	rowBuilder := parquet.NewRowBuilder(parquetSchema)
+	for i := 0; i < count; i++ {
+		rowBuilder.Reset()
+		// generate random data
+		recordValue := RecordBegin().
+			SetInt64("ID", 1+int64(i)).
+			SetInt64("CreatedAt", 2+2*int64(i)).
+			SetRecord("Person",
+				RecordBegin().
+					SetString("zName", fmt.Sprintf("john_%d", i)).
+					SetStringList("emails",
+						fmt.Sprintf("john_%d@a.com", i),
+						fmt.Sprintf("john_%d@b.com", i),
+						fmt.Sprintf("john_%d@c.com", i),
+						fmt.Sprintf("john_%d@d.com", i),
+						fmt.Sprintf("john_%d@e.com", i)).
+				RecordEnd()).
+			SetString("Company", fmt.Sprintf("company_%d", i)).
+		RecordEnd()
+		AddRecordValue(rowBuilder, recordType, parquetLevels, recordValue)
+
+		if count < 10 {
+			fmt.Printf("RecordValue: %v\n", recordValue)
+		}
+
+		row := rowBuilder.Row()
+
+		if count < 10 {
+			fmt.Printf("Row: %+v\n", row)
+		}
+
+		if err != nil {
+			t.Fatalf("rowBuilder.Build failed: %v", err)
+		}
+
+		if _, err = writer.WriteRows([]parquet.Row{row}); err != nil {
+			t.Fatalf("writer.Write failed: %v", err)
+		}
+	}
+	if err = writer.Close(); err != nil {
+		t.Fatalf("writer.WriteStop failed: %v", err)
+	}
+}
+
+func testReadingParquetFile(t *testing.T, filename string, parquetSchema *parquet.Schema, recordType *schema_pb.RecordType) (total int) {
+	parquetLevels, err := ToParquetLevels(recordType)
+	if err != nil {
+		t.Fatalf("ToParquetLevels failed: %v", err)
+	}
+
+	// read the parquet file
+	file, err := os.Open(filename)
+	if err != nil {
+		t.Fatalf("os.Open failed: %v", err)
+	}
+	defer file.Close()
+	reader := parquet.NewReader(file, parquetSchema)
+	rows := make([]parquet.Row, 128)
+	for {
+		rowCount, err := reader.ReadRows(rows)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			t.Fatalf("reader.Read failed: %v", err)
+		}
+		for i := 0; i < rowCount; i++ {
+			row := rows[i]
+			// convert parquet row to schema_pb.RecordValue
+			recordValue, err := ToRecordValue(recordType, parquetLevels, row)
+			if err != nil {
+				t.Fatalf("ToRecordValue failed: %v", err)
+			}
+			if rowCount < 10 {
+				fmt.Printf("RecordValue: %v\n", recordValue)
+			}
+		}
+		total += rowCount
+	}
+	fmt.Printf("total: %v\n", total)
+	return
+}

weed/mq/sub_coordinator/consumer_group.go

@@ -103,22 +103,22 @@ func (cg *ConsumerGroup) RebalanceConsumberGroupInstances(knownPartitionSlotToBr
 			partitionSlots = make([]*PartitionSlotToConsumerInstance, 0)
 		}
 		consumerGroupInstance.Partitions = ToPartitions(partitionSlotToBrokerList.RingSize, partitionSlots)
-		assignedPartitions := make([]*mq_pb.SubscriberToSubCoordinatorResponse_AssignedPartition, len(partitionSlots))
+		assignedPartitions := make([]*mq_pb.BrokerPartitionAssignment, len(partitionSlots))
 		for i, partitionSlot := range partitionSlots {
-			assignedPartitions[i] = &mq_pb.SubscriberToSubCoordinatorResponse_AssignedPartition{
+			assignedPartitions[i] = &mq_pb.BrokerPartitionAssignment{
 				Partition: &mq_pb.Partition{
 					RangeStop:  partitionSlot.RangeStop,
 					RangeStart: partitionSlot.RangeStart,
 					RingSize:   partitionSlotToBrokerList.RingSize,
 					UnixTimeNs: partitionSlot.UnixTimeNs,
 				},
-				Broker: partitionSlot.Broker,
+				LeaderBroker: partitionSlot.Broker,
 			}
 		}
 		response := &mq_pb.SubscriberToSubCoordinatorResponse{
 			Message: &mq_pb.SubscriberToSubCoordinatorResponse_Assignment_{
 				Assignment: &mq_pb.SubscriberToSubCoordinatorResponse_Assignment{
-					AssignedPartitions: assignedPartitions,
+					PartitionAssignments: assignedPartitions,
 				},
 			},
 		}

weed/mq/topic/local_manager.go

@@ -19,8 +19,8 @@ func NewLocalTopicManager() *LocalTopicManager {
 	}
 }
 
-// AddTopicPartition adds a topic to the local topic manager
-func (manager *LocalTopicManager) AddTopicPartition(topic Topic, localPartition *LocalPartition) {
+// AddLocalPartition adds a topic to the local topic manager
+func (manager *LocalTopicManager) AddLocalPartition(topic Topic, localPartition *LocalPartition) {
 	localTopic, ok := manager.topics.Get(topic.String())
 	if !ok {
 		localTopic = NewLocalTopic(topic)
@@ -34,8 +34,8 @@ func (manager *LocalTopicManager) AddTopicPartition(topic Topic, localPartition
 	localTopic.Partitions = append(localTopic.Partitions, localPartition)
 }
 
-// GetTopicPartition gets a topic from the local topic manager
-func (manager *LocalTopicManager) GetTopicPartition(topic Topic, partition Partition) *LocalPartition {
+// GetLocalPartition gets a topic from the local topic manager
+func (manager *LocalTopicManager) GetLocalPartition(topic Topic, partition Partition) *LocalPartition {
 	localTopic, ok := manager.topics.Get(topic.String())
 	if !ok {
 		return nil
@@ -48,7 +48,7 @@ func (manager *LocalTopicManager) RemoveTopic(topic Topic) {
 	manager.topics.Remove(topic.String())
 }
 
-func (manager *LocalTopicManager) RemoveTopicPartition(topic Topic, partition Partition) (removed bool) {
+func (manager *LocalTopicManager) RemoveLocalPartition(topic Topic, partition Partition) (removed bool) {
 	localTopic, ok := manager.topics.Get(topic.String())
 	if !ok {
 		return false
@@ -96,8 +96,9 @@ func (manager *LocalTopicManager) CollectStats(duration time.Duration) *mq_pb.Br
 					Namespace: string(localTopic.Namespace),
 					Name:      localTopic.Name,
 				},
-				Partition:     localPartition.Partition.ToPbPartition(),
-				ConsumerCount: localPartition.ConsumerCount,
+				Partition:       localPartition.Partition.ToPbPartition(),
+				PublisherCount:  int32(localPartition.Publishers.Size()),
+				SubscriberCount: int32(localPartition.Subscribers.Size()),
 			}
 			// fmt.Printf("collect topic %+v partition %+v\n", topicPartition, localPartition.Partition)
 		}

weed/mq/topic/local_partition.go

@@ -1,42 +1,74 @@
 package topic
 
 import (
+	"context"
 	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"sync"
 	"sync/atomic"
 	"time"
 )
 
 type LocalPartition struct {
+	ListenersWaits   int64
+	AckTsNs		  	 int64
+
+	// notifying clients
+	ListenersLock sync.Mutex
+	ListenersCond *sync.Cond
+
 	Partition
-	isLeader        bool
-	FollowerBrokers []pb.ServerAddress
 	LogBuffer       *log_buffer.LogBuffer
-	ConsumerCount   int32
 	Publishers      *LocalPartitionPublishers
 	Subscribers     *LocalPartitionSubscribers
-	FollowerId      int32
+
+	followerStream         mq_pb.SeaweedMessaging_PublishFollowMeClient
+	followerGrpcConnection *grpc.ClientConn
+	follower               string
 }
 
 var TIME_FORMAT = "2006-01-02-15-04-05"
 
-func NewLocalPartition(partition Partition, isLeader bool, followerBrokers []pb.ServerAddress, logFlushFn log_buffer.LogFlushFuncType, readFromDiskFn log_buffer.LogReadFromDiskFuncType) *LocalPartition {
-	return &LocalPartition{
+func NewLocalPartition(partition Partition, logFlushFn log_buffer.LogFlushFuncType, readFromDiskFn log_buffer.LogReadFromDiskFuncType) *LocalPartition {
+	lp := &LocalPartition{
 		Partition:       partition,
-		isLeader:        isLeader,
-		FollowerBrokers: followerBrokers,
-		LogBuffer: log_buffer.NewLogBuffer(fmt.Sprintf("%d/%04d-%04d", partition.UnixTimeNs, partition.RangeStart, partition.RangeStop),
-			2*time.Minute, logFlushFn, readFromDiskFn, func() {}),
 		Publishers:  NewLocalPartitionPublishers(),
 		Subscribers: NewLocalPartitionSubscribers(),
 	}
+	lp.ListenersCond = sync.NewCond(&lp.ListenersLock)
+	lp.LogBuffer = log_buffer.NewLogBuffer(fmt.Sprintf("%d/%04d-%04d", partition.UnixTimeNs, partition.RangeStart, partition.RangeStop),
+		2*time.Minute, logFlushFn, readFromDiskFn, func() {
+			if atomic.LoadInt64(&lp.ListenersWaits) > 0 {
+				lp.ListenersCond.Broadcast()
+			}
+		})
+	return lp
 }
 
-func (p *LocalPartition) Publish(message *mq_pb.DataMessage) {
-	p.LogBuffer.AddToBuffer(message.Key, message.Value, time.Now().UnixNano())
+func (p *LocalPartition) Publish(message *mq_pb.DataMessage) error {
+	p.LogBuffer.AddToBuffer(message)
+
+	// maybe send to the follower
+	if p.followerStream != nil {
+		// println("recv", string(message.Key), message.TsNs)
+		if followErr := p.followerStream.Send(&mq_pb.PublishFollowMeRequest{
+			Message: &mq_pb.PublishFollowMeRequest_Data{
+				Data: message,
+			},
+		}); followErr != nil {
+			return fmt.Errorf("send to follower %s: %v", p.follower, followErr)
+		}
+	} else {
+		atomic.StoreInt64(&p.AckTsNs, message.TsNs)
+	}
+
+	return nil
 }
 
 func (p *LocalPartition) Subscribe(clientName string, startPosition log_buffer.MessagePosition,
@@ -85,15 +117,6 @@ func (p *LocalPartition) GetEarliestInMemoryMessagePosition() log_buffer.Message
 	return p.LogBuffer.GetEarliestPosition()
 }
 
-func FromPbBrokerPartitionAssignment(self pb.ServerAddress, partition Partition, assignment *mq_pb.BrokerPartitionAssignment, logFlushFn log_buffer.LogFlushFuncType, readFromDiskFn log_buffer.LogReadFromDiskFuncType) *LocalPartition {
-	isLeader := assignment.LeaderBroker == string(self)
-	followers := make([]pb.ServerAddress, len(assignment.FollowerBrokers))
-	for i, followerBroker := range assignment.FollowerBrokers {
-		followers[i] = pb.ServerAddress(followerBroker)
-	}
-	return NewLocalPartition(partition, isLeader, followers, logFlushFn, readFromDiskFn)
-}
-
 func (p *LocalPartition) closePublishers() {
 	p.Publishers.SignalShutdown()
 }
@@ -103,18 +126,93 @@ func (p *LocalPartition) closeSubscribers() {
 
 func (p *LocalPartition) WaitUntilNoPublishers() {
 	for {
-		if p.Publishers.IsEmpty() {
+		if p.Publishers.Size() == 0 {
 			return
 		}
 		time.Sleep(113 * time.Millisecond)
 	}
 }
 
+func (p *LocalPartition) MaybeConnectToFollowers(initMessage *mq_pb.PublishMessageRequest_InitMessage, grpcDialOption grpc.DialOption) (err error) {
+	if p.followerStream != nil {
+		return nil
+	}
+	if len(initMessage.FollowerBrokers) == 0 {
+		return nil
+	}
+
+	p.follower = initMessage.FollowerBrokers[0]
+	ctx := context.Background()
+	p.followerGrpcConnection, err = pb.GrpcDial(ctx, p.follower, true, grpcDialOption)
+	if err != nil {
+		return fmt.Errorf("fail to dial %s: %v", p.follower, err)
+	}
+	followerClient := mq_pb.NewSeaweedMessagingClient(p.followerGrpcConnection)
+	p.followerStream, err = followerClient.PublishFollowMe(ctx)
+	if err != nil {
+		return fmt.Errorf("fail to create publish client: %v", err)
+	}
+	if err = p.followerStream.Send(&mq_pb.PublishFollowMeRequest{
+		Message: &mq_pb.PublishFollowMeRequest_Init{
+			Init: &mq_pb.PublishFollowMeRequest_InitMessage{
+				Topic:      initMessage.Topic,
+				Partition:  initMessage.Partition,
+			},
+		},
+	}); err != nil {
+		return err
+	}
+
+	// start receiving ack from follower
+	go func() {
+		defer func() {
+			// println("stop receiving ack from follower")
+		}()
+
+		for {
+			ack, err := p.followerStream.Recv()
+			if err != nil {
+				e, _ := status.FromError(err)
+				if e.Code() == codes.Canceled {
+					glog.V(0).Infof("local partition %v follower %v stopped", p.Partition, p.follower)
+					return
+				}
+				glog.Errorf("Receiving local partition %v  follower %s ack: %v", p.Partition, p.follower, err)
+				return
+			}
+			atomic.StoreInt64(&p.AckTsNs, ack.AckTsNs)
+			// println("recv ack", ack.AckTsNs)
+		}
+	}()
+	return nil
+}
+
 func (p *LocalPartition) MaybeShutdownLocalPartition() (hasShutdown bool) {
-	if p.Publishers.IsEmpty() && p.Subscribers.IsEmpty() {
+
+	if p.Publishers.Size() == 0 && p.Subscribers.Size() == 0 {
 		p.LogBuffer.ShutdownLogBuffer()
+		for !p.LogBuffer.IsAllFlushed() {
+			time.Sleep(113 * time.Millisecond)
+		}
+		if p.followerStream != nil {
+			// send close to the follower
+			if followErr := p.followerStream.Send(&mq_pb.PublishFollowMeRequest{
+				Message: &mq_pb.PublishFollowMeRequest_Close{
+					Close: &mq_pb.PublishFollowMeRequest_CloseMessage{},
+				},
+			}); followErr != nil {
+				glog.Errorf("Error closing follower stream: %v", followErr)
+			}
+			glog.V(4).Infof("closing grpcConnection to follower")
+			p.followerGrpcConnection.Close()
+			p.followerStream = nil
+			p.follower = ""
+		}
+
 		hasShutdown = true
 	}
+
+	glog.V(0).Infof("local partition %v Publisher:%d Subscriber:%d follower:%s shutdown %v", p.Partition, p.Publishers.Size(), p.Subscribers.Size(), p.follower, hasShutdown)
 	return
 }
 
@@ -122,5 +220,20 @@ func (p *LocalPartition) Shutdown() {
 	p.closePublishers()
 	p.closeSubscribers()
 	p.LogBuffer.ShutdownLogBuffer()
-	atomic.StoreInt32(&p.FollowerId, 0)
+	glog.V(0).Infof("local partition %v shutting down", p.Partition)
+}
+
+func (p *LocalPartition) NotifyLogFlushed(flushTsNs int64) {
+	if p.followerStream != nil {
+		if followErr := p.followerStream.Send(&mq_pb.PublishFollowMeRequest{
+			Message: &mq_pb.PublishFollowMeRequest_Flush{
+				Flush: &mq_pb.PublishFollowMeRequest_FlushMessage{
+					TsNs: flushTsNs,
+				},
+			},
+		}); followErr != nil {
+			glog.Errorf("send follower %s flush message: %v", p.follower, followErr)
+		}
+		// println("notifying", p.follower, "flushed at", flushTsNs)
+	}
 }

weed/mq/topic/local_partition_publishers.go

@@ -44,13 +44,6 @@ func (p *LocalPartitionPublishers) SignalShutdown() {
 	}
 }
 
-func (p *LocalPartitionPublishers) IsEmpty() bool {
-	p.publishersLock.RLock()
-	defer p.publishersLock.RUnlock()
-
-	return len(p.publishers) == 0
-}
-
 func (p *LocalPartitionPublishers) Size() int {
 	p.publishersLock.RLock()
 	defer p.publishersLock.RUnlock()

weed/mq/topic/local_partition_subscribers.go

@@ -48,13 +48,6 @@ func (p *LocalPartitionSubscribers) SignalShutdown() {
 	}
 }
 
-func (p *LocalPartitionSubscribers) IsEmpty() bool {
-	p.SubscribersLock.RLock()
-	defer p.SubscribersLock.RUnlock()
-
-	return len(p.Subscribers) == 0
-}
-
 func (p *LocalPartitionSubscribers) Size() int {
 	p.SubscribersLock.RLock()
 	defer p.SubscribersLock.RUnlock()

weed/operation/assign_file_id.go

@@ -6,6 +6,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"google.golang.org/grpc"
 	"sync"
@@ -193,6 +194,7 @@ func Assign(masterFn GetMasterFn, grpcDialOption grpc.DialOption, primaryRequest
 		})
 
 		if lastError != nil {
+			stats.FilerHandlerCounter.WithLabelValues(stats.ErrorChunkAssign).Inc()
 			continue
 		}
 
@@ -262,6 +264,7 @@ func (so *StorageOption) ToAssignRequests(count int) (ar *VolumeAssignRequest, a
 		WritableVolumeCount: so.VolumeGrowthCount,
 	}
 	if so.DataCenter != "" || so.Rack != "" || so.DataNode != "" {
+		ar.WritableVolumeCount = uint32(count)
 		altRequest = &VolumeAssignRequest{
 			Count:               uint64(count),
 			Replication:         so.Replication,

weed/operation/upload_content.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/valyala/bytebufferpool"
 	"io"
 	"mime"
 	"mime/multipart"
@@ -32,6 +33,7 @@ type UploadOption struct {
 	Jwt               security.EncodedJwt
 	RetryForever      bool
 	Md5               string
+	BytesBuffer       *bytes.Buffer
 }
 
 type UploadResult struct {
@@ -261,6 +263,7 @@ func doUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult
 			PairMap:           option.PairMap,
 			Jwt:               option.Jwt,
 			Md5:               option.Md5,
+			BytesBuffer:       option.BytesBuffer,
 		})
 		if uploadResult == nil {
 			return
@@ -275,9 +278,17 @@ func doUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult
 }
 
 func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize int, option *UploadOption) (*UploadResult, error) {
-	buf := GetBuffer()
-	defer PutBuffer(buf)
-	body_writer := multipart.NewWriter(buf)
+	var body_writer *multipart.Writer
+	var reqReader *bytes.Reader
+	var buf *bytebufferpool.ByteBuffer
+	if option.BytesBuffer == nil {
+		buf = GetBuffer()
+		defer PutBuffer(buf)
+		body_writer = multipart.NewWriter(buf)
+	} else {
+		option.BytesBuffer.Reset()
+		body_writer = multipart.NewWriter(option.BytesBuffer)
+	}
 	h := make(textproto.MIMEHeader)
 	filename := fileNameEscaper.Replace(option.Filename)
 	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename))
@@ -309,8 +320,12 @@ func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize
 		glog.V(0).Infoln("error closing body", err)
 		return nil, err
 	}
-
-	req, postErr := http.NewRequest("POST", option.UploadUrl, bytes.NewReader(buf.Bytes()))
+	if option.BytesBuffer == nil {
+		reqReader = bytes.NewReader(buf.Bytes())
+	} else {
+		reqReader = bytes.NewReader(option.BytesBuffer.Bytes())
+	}
+	req, postErr := http.NewRequest("POST", option.UploadUrl, reqReader)
 	if postErr != nil {
 		glog.V(1).Infof("create upload request %s: %v", option.UploadUrl, postErr)
 		return nil, fmt.Errorf("create upload request %s: %v", option.UploadUrl, postErr)