4.19

* filer: bootstrap pre-existing metadata when a new filer joins a cluster

When a filer connects to a peer for the first time (no stored sync
offset), it now does a full BFS traversal of the peer's metadata via
TraverseBfsMetadata before starting the incremental change stream.
This ensures filer2 sees all data that existed before it started,
fixing the issue where only post-startup changes were synced.

Closes #8961

* filer: upsert during bootstrap and persist offset immediately

- Use upsert (insert, then update on conflict) during metadata
  traversal so the bootstrap doesn't fail on the root directory
  or after a partial previous attempt.
- Persist the sync offset right after a successful traversal so
  a retry doesn't redo the full BFS.

* filer: address review feedback on metadata bootstrap

- Use peer-side max Mtime as the streaming cursor instead of local
  time.Now() to avoid missing events due to clock skew between filers.
  traversePeerMetadata now returns the high-water Mtime (nanoseconds)
  observed during BFS traversal.

- Compare Mtime before overwriting during bootstrap: if a local entry
  is newer than the peer's version, skip the update instead of
  clobbering it.

- Only trigger full BFS traversal on ErrKvNotFound (key genuinely
  missing). Transient KvGet errors (connection issues, etc.) are now
  propagated instead of silently falling through to a full re-sync.
  Changed readOffset to use %w so errors.Is works through the chain.

* filer: address review findings on bootstrap sync

- Use wall-clock time with safety margin for stream cursor instead of
  entry Mtime. Mtime is file modification time (can be arbitrary),
  while the metadata stream uses TsNs (event log time). Using
  time.Now() minus 1 minute before traversal ensures no events are
  missed even with clock skew, matching the proven filer.meta.backup
  pattern.

- Pass ExcludedPrefixes=[SystemLogDir] to TraverseBfsMetadata so
  the server prunes internal log entries server-side instead of
  transferring them over the network only to be filtered client-side.

- Fail fast if updateOffset fails after bootstrap. If we can't
  persist the offset, bail out rather than proceeding and potentially
  losing the expensive BFS work on the next retry.

.claude/scheduled_tasks.lock

@@ -0,0 +1 @@
+{"sessionId":"d6574c47-eafc-4a94-9dce-f9ffea22b53c","pid":10111,"acquiredAt":1775248373916}

.github/workflows/container_dev.yml

@@ -9,17 +9,96 @@ permissions:
 
 jobs:
 
+  # ── Pre-build Rust volume server binaries natively ──────────────────
+  build-rust-binaries:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - target: x86_64-unknown-linux-musl
+            arch: amd64
+          - target: aarch64-unknown-linux-musl
+            arch: arm64
+            cross: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - name: Install musl tools (amd64)
+        if: ${{ !matrix.cross }}
+        run: sudo apt-get install -y musl-tools
+
+      - name: Install cross-compilation tools (arm64)
+        if: matrix.cross
+        run: |
+          sudo apt-get install -y gcc-aarch64-linux-gnu
+          echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
+          # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
+          echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-docker-dev-${{ matrix.target }}-
+
+      - name: Build normal variant
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }} --no-default-features
+          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: rust-volume-${{ matrix.arch }}
+          path: weed-volume-normal-${{ matrix.arch }}
+
   build-dev-containers:
+    needs: [build-rust-binaries]
     runs-on: [ubuntu-latest]
 
     steps:
-      -
-        name: Checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
-      -
-        name: Docker meta
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Download pre-built Rust binaries
+        uses: actions/download-artifact@v8
+        with:
+          pattern: rust-volume-*
+          merge-multiple: true
+          path: ./rust-bins
+
+      - name: Place Rust binaries in Docker context
+        run: |
+          mkdir -p docker/weed-volume-prebuilt
+          for arch in amd64 arm64; do
+            src="./rust-bins/weed-volume-normal-${arch}"
+            if [ -f "$src" ]; then
+              cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
+              echo "Placed pre-built Rust binary for ${arch}"
+            fi
+          done
+          ls -la docker/weed-volume-prebuilt/
+
+      - name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v3
+        uses: docker/metadata-action@v6
         with:
           images: |
             chrislusf/seaweedfs
@@ -30,40 +109,40 @@ jobs:
             org.opencontainers.image.title=seaweedfs
             org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
             org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
-      -
-        name: Create BuildKit config
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Create BuildKit config
         run: |
           cat > /tmp/buildkitd.toml <<EOF
           [registry."docker.io"]
             mirrors = ["https://mirror.gcr.io"]
           EOF
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
         with:
           buildkitd-flags: "--debug"
           buildkitd-config: /tmp/buildkitd.toml
-      -
-        name: Login to Docker Hub
+
+      - name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
+
+      - name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
+        uses: docker/login-action@v4.1.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
+
+      - name: Build
+        uses: docker/build-push-action@v7
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_foundationdb_version.yml

@@ -133,7 +133,7 @@ jobs:
 
       - name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/container_latest.yml

@@ -23,6 +23,11 @@ on:
           - all
           - standard
           - large_disk
+      publish:
+        description: 'Publish images and manifests'
+        required: true
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -33,6 +38,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       variants: ${{ steps.set-variants.outputs.variants }}
+      publish: ${{ steps.set-publish.outputs.publish }}
     steps:
       - name: Select variants for this run
         id: set-variants
@@ -43,9 +49,89 @@ jobs:
             variants='["standard","large_disk"]'
           fi
           echo "variants=$variants" >> "$GITHUB_OUTPUT"
+      - name: Select publish mode
+        id: set-publish
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
+          else
+            echo "publish=true" >> "$GITHUB_OUTPUT"
+          fi
+
+  # ── Pre-build Rust volume server binaries natively ──────────────────
+  build-rust-binaries:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - target: x86_64-unknown-linux-musl
+            arch: amd64
+          - target: aarch64-unknown-linux-musl
+            arch: arm64
+            cross: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
+
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - name: Install musl tools (amd64)
+        if: ${{ !matrix.cross }}
+        run: sudo apt-get install -y musl-tools
+
+      - name: Install cross-compilation tools (arm64)
+        if: matrix.cross
+        run: |
+          sudo apt-get install -y gcc-aarch64-linux-gnu
+          echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
+          # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
+          echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-docker-${{ matrix.target }}-
+
+      - name: Build large-disk variant
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }}
+          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
+
+      - name: Build normal variant
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }} --no-default-features
+          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: rust-volume-${{ matrix.arch }}
+          path: |
+            weed-volume-large-disk-${{ matrix.arch }}
+            weed-volume-normal-${{ matrix.arch }}
 
   build:
-    needs: [setup]
+    needs: [setup, build-rust-binaries]
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -80,11 +166,32 @@ jobs:
           if [ "${{ matrix.variant }}" == "large_disk" ]; then
             echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
             echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
+            echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
           else
             echo "tag_suffix=" >> $GITHUB_OUTPUT
             echo "build_args=" >> $GITHUB_OUTPUT
+            echo "rust_variant=normal" >> $GITHUB_OUTPUT
           fi
 
+      - name: Download pre-built Rust binaries
+        uses: actions/download-artifact@v8
+        with:
+          pattern: rust-volume-*
+          merge-multiple: true
+          path: ./rust-bins
+
+      - name: Place Rust binaries in Docker context
+        run: |
+          mkdir -p docker/weed-volume-prebuilt
+          for arch in amd64 arm64; do
+            src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
+            if [ -f "$src" ]; then
+              cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
+              echo "Placed pre-built Rust binary for ${arch}"
+            fi
+          done
+          ls -la docker/weed-volume-prebuilt/
+
       - name: Docker meta
         id: docker_meta
         uses: docker/metadata-action@v6
@@ -112,14 +219,14 @@ jobs:
           buildkitd-flags: "--debug"
           buildkitd-config: /tmp/buildkitd.toml
       - name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4
+        if: needs.setup.outputs.publish == 'true'
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Login to GHCR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4
+        if: needs.setup.outputs.publish == 'true'
+        uses: docker/login-action@v4.1.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
@@ -130,7 +237,7 @@ jobs:
           DOCKER_BUILDKIT: 1
         with:
           context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ needs.setup.outputs.publish == 'true' }}
           file: ./docker/Dockerfile.go_build
           platforms: linux/${{ matrix.platform }}
           # Push to GHCR only during build to avoid Docker Hub rate limits
@@ -152,7 +259,7 @@ jobs:
 
   trivy-scan:
     runs-on: ubuntu-latest
-    needs: [setup, build]
+    needs: [setup, build, build-rust-binaries]
     strategy:
       matrix:
         variant: ${{ fromJSON(needs.setup.outputs.variants) }}
@@ -166,33 +273,135 @@ jobs:
             echo "tag_suffix=" >> $GITHUB_OUTPUT
           fi
       - name: Login to GHCR
-        uses: docker/login-action@v4
+        if: needs.setup.outputs.publish == 'true'
+        uses: docker/login-action@v4.1.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
-      - name: Run Trivy vulnerability scanner
-        # Pin to SHA — mutable tags were compromised (GHSA-69fq-xp46-6x23)
+      - name: Checkout for local scan build
+        if: needs.setup.outputs.publish != 'true'
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
+      - name: Download pre-built Rust binaries for local scan
+        if: needs.setup.outputs.publish != 'true'
+        uses: actions/download-artifact@v8
+        with:
+          pattern: rust-volume-*
+          merge-multiple: true
+          path: ./rust-bins
+      - name: Place Rust binaries in Docker context for local scan
+        if: needs.setup.outputs.publish != 'true'
+        run: |
+          rust_variant="normal"
+          if [ "${{ matrix.variant }}" == "large_disk" ]; then
+            rust_variant="large-disk"
+          fi
+          mkdir -p docker/weed-volume-prebuilt
+          for arch in amd64 arm64; do
+            src="./rust-bins/weed-volume-${rust_variant}-${arch}"
+            if [ -f "$src" ]; then
+              cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
+              echo "Placed pre-built Rust binary for ${arch}"
+            fi
+          done
+          ls -la docker/weed-volume-prebuilt/
+      - name: Create BuildKit config for local scan build
+        if: needs.setup.outputs.publish != 'true'
+        run: |
+          cat > /tmp/buildkitd.toml <<EOF
+          [registry."docker.io"]
+            mirrors = ["https://mirror.gcr.io"]
+          EOF
+      - name: Set up Docker Buildx for local scan build
+        if: needs.setup.outputs.publish != 'true'
+        uses: docker/setup-buildx-action@v4
+        with:
+          buildkitd-flags: "--debug"
+          buildkitd-config: /tmp/buildkitd.toml
+      - name: Build local scan image tarball
+        if: needs.setup.outputs.publish != 'true'
+        uses: docker/build-push-action@v7
+        env:
+          DOCKER_BUILDKIT: 1
+        with:
+          context: ./docker
+          file: ./docker/Dockerfile.go_build
+          platforms: linux/amd64
+          outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1
+            BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
+            ${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
+      - name: Trivy report (published image)
+        if: needs.setup.outputs.publish == 'true'
+        # Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
         uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
         with:
-          # Scan amd64 only — OS packages are identical across architectures
+          scan-type: image
+          # Scan amd64 only - OS packages are identical across architectures
           # since they all use the same alpine base, so a single-arch scan
           # provides sufficient coverage without multiplying CI time.
           image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
+          scanners: vuln
+          vuln-type: os,library
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          limit-severities-for-sarif: true
           format: sarif
           output: trivy-results.sarif
+          exit-code: '0'
+      - name: Trivy report (local tarball)
+        if: needs.setup.outputs.publish != 'true'
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
+          scanners: vuln
+          vuln-type: os,library
           severity: HIGH,CRITICAL
-          exit-code: '1'
+          ignore-unfixed: true
+          limit-severities-for-sarif: true
+          format: sarif
+          output: trivy-results.sarif
+          exit-code: '0'
       - name: Upload Trivy scan results to GitHub Security
-        uses: github/codeql-action/upload-sarif@v3
+        uses: github/codeql-action/upload-sarif@v4
         if: always()
         with:
           sarif_file: trivy-results.sarif
+      - name: Trivy gate (published image)
+        if: needs.setup.outputs.publish == 'true'
+        # Gate only on fixable high/critical vulnerabilities. Non-fixable
+        # findings are still visible in the SARIF upload above.
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          scan-type: image
+          image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
+          scanners: vuln
+          vuln-type: os,library
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          format: table
+          exit-code: '1'
+          skip-setup-trivy: true
+      - name: Trivy gate (local tarball)
+        if: needs.setup.outputs.publish != 'true'
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
+          scanners: vuln
+          vuln-type: os,library
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          format: table
+          exit-code: '1'
+          skip-setup-trivy: true
 
   create-manifest:
     runs-on: ubuntu-latest
     needs: [setup, build, trivy-scan]
-    if: github.event_name != 'pull_request'
+    if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
     strategy:
       matrix:
         variant: ${{ fromJSON(needs.setup.outputs.variants) }}
@@ -220,12 +429,12 @@ jobs:
             ghcr.io/chrislusf/seaweedfs
           tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
       - name: Login to Docker Hub
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Login to GHCR
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}

.github/workflows/container_release_foundationdb.yml

@@ -42,7 +42,7 @@ jobs:
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/container_release_unified.yml

@@ -39,7 +39,82 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+
+  # ── Pre-build Rust volume server binaries natively ──────────────────
+  # Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour
+  # emulated cargo build into ~15 minutes of native compilation.
+  build-rust-binaries:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - target: x86_64-unknown-linux-musl
+            arch: amd64
+          - target: aarch64-unknown-linux-musl
+            arch: arm64
+            cross: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - name: Install musl tools (amd64)
+        if: ${{ !matrix.cross }}
+        run: sudo apt-get install -y musl-tools
+
+      - name: Install cross-compilation tools (arm64)
+        if: matrix.cross
+        run: |
+          sudo apt-get install -y gcc-aarch64-linux-gnu
+          echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
+          # Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
+          echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-docker-${{ matrix.target }}-
+
+      - name: Build large-disk variant
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }}
+          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
+
+      - name: Build normal variant
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }} --no-default-features
+          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: rust-volume-${{ matrix.arch }}
+          path: |
+            weed-volume-large-disk-${{ matrix.arch }}
+            weed-volume-normal-${{ matrix.arch }}
+
+  # ── Build Docker containers ─────────────────────────────────────────
   build:
+    needs: [build-rust-binaries]
     runs-on: ubuntu-latest
     strategy:
       # Build sequentially to avoid rate limits
@@ -52,20 +127,23 @@ jobs:
             dockerfile: ./docker/Dockerfile.go_build
             build_args: ""
             tag_suffix: ""
-            
-          # Large disk - multi-arch  
+            rust_variant: normal
+
+          # Large disk - multi-arch
           - variant: large_disk
             platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
             dockerfile: ./docker/Dockerfile.go_build
             build_args: TAGS=5BytesOffset
             tag_suffix: _large_disk
-            
+            rust_variant: large-disk
+
           # Full tags - multi-arch
           - variant: full
             platforms: linux/amd64,linux/arm64
             dockerfile: ./docker/Dockerfile.go_build
             build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
             tag_suffix: _full
+            rust_variant: normal
 
           # Large disk + full tags - multi-arch
           - variant: large_disk_full
@@ -73,19 +151,42 @@ jobs:
             dockerfile: ./docker/Dockerfile.go_build
             build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
             tag_suffix: _large_disk_full
-            
+            rust_variant: large-disk
+
           # RocksDB large disk - amd64 only
           - variant: rocksdb
             platforms: linux/amd64
             dockerfile: ./docker/Dockerfile.rocksdb_large
             build_args: ""
             tag_suffix: _large_disk_rocksdb
+            rust_variant: large-disk
             
     steps:
       - name: Checkout
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         uses: actions/checkout@v6
-        
+
+      - name: Download pre-built Rust binaries
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+        uses: actions/download-artifact@v8
+        with:
+          pattern: rust-volume-*
+          merge-multiple: true
+          path: ./rust-bins
+
+      - name: Place Rust binaries in Docker context
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+        run: |
+          mkdir -p docker/weed-volume-prebuilt
+          for arch in amd64 arm64; do
+            src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
+            if [ -f "$src" ]; then
+              cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
+              echo "Placed pre-built Rust binary for ${arch}"
+            fi
+          done
+          ls -la docker/weed-volume-prebuilt/
+
       - name: Free Disk Space
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
         run: |
@@ -135,14 +236,14 @@ jobs:
           
       - name: Login to Docker Hub
         if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           
       - name: Login to GHCR
         if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
@@ -198,14 +299,14 @@ jobs:
     steps:
       - name: Login to Docker Hub
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           
       - name: Login to GHCR
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
-        uses: docker/login-action@v4
+        uses: docker/login-action@v4.1.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}

.github/workflows/container_rocksdb_version.yml

@@ -88,7 +88,7 @@ jobs:
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
 
       - name: Login to Docker Hub
-        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
+        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/helm_ci.yml

@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v4
+        uses: azure/setup-helm@v5
         with:
           version: v3.18.4
 

.github/workflows/rust-volume-server-tests.yml

@@ -43,7 +43,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry
@@ -80,7 +80,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry
@@ -93,7 +93,7 @@ jobs:
       - name: Build Go weed binary
         run: |
           cd weed
-          go build -o weed .
+          go build -tags 5BytesOffset -o weed .
           chmod +x weed
           ./weed version
 
@@ -156,7 +156,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry
@@ -169,7 +169,7 @@ jobs:
       - name: Build Go weed binary
         run: |
           cd weed
-          go build -o weed .
+          go build -tags 5BytesOffset -o weed .
           chmod +x weed
           ./weed version
 

.github/workflows/rust_binaries_dev.yml

@@ -47,7 +47,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry
@@ -118,7 +118,7 @@ jobs:
           targets: ${{ matrix.target }}
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry

.github/workflows/rust_binaries_release.yml

@@ -39,11 +39,19 @@ jobs:
       - name: Install cross-compilation tools
         if: matrix.cross
         run: |
-          sudo apt-get install -y gcc-aarch64-linux-gnu
+          sudo dpkg --add-architecture arm64
+          sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
+          echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
+          echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
+          sudo apt-get update
+          sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
           echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
+          echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"
+          echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"
+          echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry
@@ -80,6 +88,7 @@ jobs:
           rm weed-volume-normal
 
       - name: Upload release assets
+        if: startsWith(github.ref, 'refs/tags/')
         uses: softprops/action-gh-release@v2
         with:
           files: |
@@ -88,6 +97,15 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Upload artifacts
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: rust-volume-${{ matrix.asset_suffix }}
+          path: |
+            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
+            weed-volume_${{ matrix.asset_suffix }}.tar.gz
+
   build-rust-volume-darwin:
     permissions:
       contents: write
@@ -112,7 +130,7 @@ jobs:
           targets: ${{ matrix.target }}
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry
@@ -147,6 +165,7 @@ jobs:
           rm weed-volume-normal
 
       - name: Upload release assets
+        if: startsWith(github.ref, 'refs/tags/')
         uses: softprops/action-gh-release@v2
         with:
           files: |
@@ -155,6 +174,15 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Upload artifacts
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: rust-volume-${{ matrix.asset_suffix }}
+          path: |
+            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
+            weed-volume_${{ matrix.asset_suffix }}.tar.gz
+
   build-rust-volume-windows:
     permissions:
       contents: write
@@ -170,7 +198,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry
@@ -206,6 +234,7 @@ jobs:
           rm weed-volume-normal.exe
 
       - name: Upload release assets
+        if: startsWith(github.ref, 'refs/tags/')
         uses: softprops/action-gh-release@v2
         with:
           files: |
@@ -213,3 +242,12 @@ jobs:
             weed-volume_windows_amd64.zip
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload artifacts
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: rust-volume-windows_amd64
+          path: |
+            weed-volume_large_disk_windows_amd64.zip
+            weed-volume_windows_amd64.zip

.superset/config.json

@@ -0,0 +1,5 @@
+{
+  "setup": [],
+  "teardown": [],
+  "run": []
+}

docker/Dockerfile.go_build

@@ -16,15 +16,22 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
   && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
   && CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
 
-# Rust volume server builder (amd64/arm64 only)
-FROM rust:1-alpine as rust_builder
+# Rust volume server: use pre-built binary from CI when available (placed in
+# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
+# from source.  Pre-building avoids a multi-hour QEMU-emulated cargo build
+# for non-native architectures.
+FROM alpine:3.23 as rust_builder
 ARG TARGETARCH
-RUN apk add musl-dev protobuf-dev git
-COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
-COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/proto /build/proto
-WORKDIR /build/seaweed-volume
 ARG TAGS
-RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
+COPY weed-volume-prebuilt/ /prebuilt/
+COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
+COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
+WORKDIR /build/seaweed-volume
+RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
+      echo "Using pre-built Rust binary for ${TARGETARCH}" && \
+      cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \
+    elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
+      apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
       if [ "$TAGS" = "5BytesOffset" ]; then \
         cargo build --release; \
       else \
@@ -49,7 +56,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
 # To disable: docker run -e GODEBUG=fips140=off ...
 
 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse curl su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse curl su-exec libgcc && \
     addgroup -g 1000 seaweed && \
     adduser -D -u 1000 -G seaweed seaweed
 
@@ -79,9 +87,5 @@ RUN mkdir -p /data/filerldb2 && \
 VOLUME /data
 WORKDIR /data
 
-# Run as non-root by default (satisfies security scanners).
-# Use `docker run --user root` if you need the entrypoint to fix
-# /data volume ownership before dropping privileges.
-USER seaweed
-
+# Entrypoint will handle permission fixes and user switching
 ENTRYPOINT ["/entrypoint.sh"]

docker/Dockerfile.local

@@ -7,7 +7,8 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
 COPY ./entrypoint.sh /entrypoint.sh
 
 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse curl su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse curl su-exec && \
     addgroup -g 1000 seaweed && \
     adduser -D -u 1000 -G seaweed seaweed
 
@@ -37,9 +38,5 @@ RUN mkdir -p /data/filerldb2 && \
 VOLUME /data
 WORKDIR /data
 
-# Run as non-root by default (satisfies security scanners).
-# Use `docker run --user root` if you need the entrypoint to fix
-# /data volume ownership before dropping privileges.
-USER seaweed
-
+# Entrypoint will handle permission fixes and user switching
 ENTRYPOINT ["/entrypoint.sh"]

docker/Dockerfile.rocksdb_large

@@ -34,7 +34,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
 
 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse snappy gflags curl su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse snappy gflags curl su-exec && \
     addgroup -g 1000 seaweed && \
     adduser -D -u 1000 -G seaweed seaweed
 

docker/Dockerfile.rocksdb_large_local

@@ -17,7 +17,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
 
 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse snappy gflags curl tmux su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse snappy gflags curl tmux su-exec && \
     addgroup -g 1000 seaweed && \
     adduser -D -u 1000 -G seaweed seaweed
 

docker/compose/s3_static_users_example.json

@@ -0,0 +1,53 @@
+{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        {
+          "accessKey": "AKIAIOSFODNN7EXAMPLE",
+          "secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    },
+    {
+      "name": "steward",
+      "credentials": [
+        {
+          "accessKey": "steward-key",
+          "secretKey": "steward-secret"
+        }
+      ],
+      "actions": [
+        "Read",
+        "List",
+        "Write"
+      ]
+    },
+    {
+      "name": "le001",
+      "credentials": [
+        {
+          "accessKey": "le001-key",
+          "secretKey": "le001-secret"
+        }
+      ],
+      "actions": [
+        "Read",
+        "List"
+      ]
+    },
+    {
+      "name": "anonymous",
+      "actions": [
+        "Read"
+      ]
+    }
+  ]
+}

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/google/btree v1.1.3
 	github.com/google/uuid v1.6.0
 	github.com/google/wire v0.7.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.17.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.19.0 // indirect
 	github.com/gorilla/mux v1.8.1
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -83,10 +83,10 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/viant/ptrie v1.0.1
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/scram v1.2.0
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	go.etcd.io/etcd/client/v3 v3.6.7
+	go.etcd.io/etcd/client/v3 v3.6.10
 	go.mongodb.org/mongo-driver v1.17.9
 	go.opencensus.io v0.24.0 // indirect
 	gocloud.dev v0.45.0
@@ -94,27 +94,26 @@ require (
 	gocloud.dev/pubsub/rabbitpubsub v0.45.0
 	golang.org/x/crypto v0.49.0
 	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
-	golang.org/x/image v0.36.0
-	golang.org/x/net v0.51.0
+	golang.org/x/image v0.38.0
+	golang.org/x/net v0.52.0
 	golang.org/x/oauth2 v0.36.0
 	golang.org/x/sys v0.42.0
 	golang.org/x/text v0.35.0 // indirect
 	golang.org/x/tools v0.42.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
-	google.golang.org/api v0.267.0
-	google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
+	google.golang.org/api v0.274.0
+	google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 // indirect
 	google.golang.org/grpc v1.79.3
 	google.golang.org/protobuf v1.36.11
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
-	modernc.org/mathutil v1.7.1
+	modernc.org/mathutil v1.7.1 // indirect
 	modernc.org/memory v1.11.0 // indirect
 	modernc.org/sqlite v1.46.1
-	modernc.org/strutil v1.2.1
 )
 
 require (
-	cloud.google.com/go/kms v1.25.0
+	cloud.google.com/go/kms v1.26.0
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
 	github.com/Jille/raft-grpc-transport v1.6.1
 	github.com/ThreeDotsLabs/watermill v1.5.1
@@ -124,22 +123,21 @@ require (
 	github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
 	github.com/arangodb/go-driver v1.6.9
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.41.4
-	github.com/aws/aws-sdk-go-v2/config v1.32.9
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.12
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
+	github.com/aws/aws-sdk-go-v2 v1.41.5
+	github.com/aws/aws-sdk-go-v2/config v1.32.13
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.14
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
-	github.com/getsentry/sentry-go v0.43.0
+	github.com/getsentry/sentry-go v0.44.1
 	github.com/go-ldap/ldap/v3 v3.4.13
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
 	github.com/hashicorp/raft v1.7.3
 	github.com/hashicorp/raft-boltdb/v2 v2.3.1
-	github.com/hashicorp/vault/api v1.22.0
+	github.com/hashicorp/vault/api v1.23.0
 	github.com/jhump/protoreflect v1.18.0
 	github.com/linkedin/goavro/v2 v2.15.0
-	github.com/mattn/go-sqlite3 v1.14.34
 	github.com/minio/crc64nvme v1.1.1
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/parquet-go/parquet-go v0.28.0
@@ -157,7 +155,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
 	github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
-	go.etcd.io/etcd/client/pkg/v3 v3.6.7
+	go.etcd.io/etcd/client/pkg/v3 v3.6.10
 	go.uber.org/atomic v1.11.0
 	golang.org/x/sync v0.20.0
 	golang.org/x/tools/godoc v0.1.0-deprecated
@@ -181,7 +179,7 @@ require (
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
 	github.com/apache/thrift v0.22.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/bazelbuild/rules_go v0.46.0 // indirect
 	github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
@@ -291,7 +289,7 @@ require (
 
 require (
 	cel.dev/expr v0.25.1 // indirect
-	cloud.google.com/go/auth v0.18.1 // indirect
+	cloud.google.com/go/auth v0.18.2 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	cloud.google.com/go/iam v1.5.3 // indirect
@@ -323,22 +321,22 @@ require (
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.9
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
 	github.com/aws/smithy-go v1.24.2
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
@@ -371,7 +369,7 @@ require (
 	github.com/geoffgarside/ber v1.2.0 // indirect
 	github.com/go-chi/chi/v5 v5.2.5 // indirect
 	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
@@ -387,7 +385,7 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
 	github.com/gorilla/schema v1.4.1 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/gorilla/sessions v1.4.0
@@ -486,22 +484,22 @@ require (
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
 	go.etcd.io/bbolt v1.4.3 // indirect
-	go.etcd.io/etcd/api/v3 v3.6.7 // indirect
+	go.etcd.io/etcd/api/v3 v3.6.10 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
-	go.opentelemetry.io/otel v1.40.0 // indirect
-	go.opentelemetry.io/otel/metric v1.40.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.40.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
-	go.opentelemetry.io/otel/trace v1.40.0 // indirect
+	go.opentelemetry.io/otel v1.42.0 // indirect
+	go.opentelemetry.io/otel/metric v1.42.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.42.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.42.0 // indirect
+	go.opentelemetry.io/otel/trace v1.42.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
 	golang.org/x/term v0.41.0 // indirect
-	golang.org/x/time v0.14.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

go.sum

@@ -94,8 +94,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo
 cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
 cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0=
 cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
-cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
-cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
+cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
+cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0=
@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
 cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
 cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
 cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
-cloud.google.com/go/kms v1.25.0 h1:gVqvGGUmz0nYCmtoxWmdc1wli2L1apgP8U4fghPGSbQ=
-cloud.google.com/go/kms v1.25.0/go.mod h1:XIdHkzfj0bUO3E+LvwPg+oc7s58/Ns8Nd8Sdtljihbk=
+cloud.google.com/go/kms v1.26.0 h1:cK9mN2cf+9V63D3H1f6koxTatWy39aTI/hCjz1I+adU=
+cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58=
 cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
 cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
 cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
@@ -310,8 +310,8 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6
 cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
 cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
 cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
-cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
-cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
+cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA=
+cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak=
 cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
 cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
 cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
@@ -714,48 +714,48 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
-github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A=
-github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
+github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
+github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
+github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
+github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 h1:foqo/ocQ7WqKwy3FojGtZQJo0FR4vto9qnz9VaumbCo=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
 github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
 github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
 github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
 github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
@@ -1051,8 +1051,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
 github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
 github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
 github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
-github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
-github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
+github.com/getsentry/sentry-go v0.44.1 h1:/cPtrA5qB7uMRrhgSn9TYtcEF36auGP3Y6+ThvD/yaI=
+github.com/getsentry/sentry-go v0.44.1/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
 github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
@@ -1076,8 +1076,8 @@ github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
-github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
+github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -1284,8 +1284,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY
 github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
 github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
 github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
-github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
-github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
+github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
+github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
@@ -1296,8 +1296,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK
 github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
 github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
 github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
-github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
-github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
+github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE=
+github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
 github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
 github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
@@ -1383,8 +1383,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKc
 github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0=
 github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc=
 github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE=
-github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0=
-github.com/hashicorp/vault/api v1.22.0/go.mod h1:IUZA2cDvr4Ok3+NtK2Oq/r+lJeXkeCrHRmqdyWfpmGM=
+github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A=
+github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -1838,8 +1838,6 @@ github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1Ivohy
 github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
 github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564 h1:TgxPraf1NmF6XTcUG53ULpLQrKvhtUJxQ3hyekxSDNQ=
 github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
-github.com/seaweedfs/go-fuse/v2 v2.9.1 h1:gnKmfrKreCRGJmekGz5WMnNZqXEf9s9+V2hdWQdvx88=
-github.com/seaweedfs/go-fuse/v2 v2.9.1/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
 github.com/seaweedfs/go-fuse/v2 v2.9.2 h1:IfP/yFjLGO4rALcJY2Gb39PlebHxLnj7dkIiQAjFres=
 github.com/seaweedfs/go-fuse/v2 v2.9.2/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
 github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
@@ -2038,8 +2036,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
-github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
+github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -2103,12 +2101,12 @@ go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
 go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
 go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
-go.etcd.io/etcd/api/v3 v3.6.7 h1:7BNJ2gQmc3DNM+9cRkv7KkGQDayElg8x3X+tFDYS+E0=
-go.etcd.io/etcd/api/v3 v3.6.7/go.mod h1:xJ81TLj9hxrYYEDmXTeKURMeY3qEDN24hqe+q7KhbnI=
-go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/AhBVs=
-go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
-go.etcd.io/etcd/client/v3 v3.6.7 h1:9WqA5RpIBtdMxAy1ukXLAdtg2pAxNqW5NUoO2wQrE6U=
-go.etcd.io/etcd/client/v3 v3.6.7/go.mod h1:2XfROY56AXnUqGsvl+6k29wrwsSbEh1lAouQB1vHpeE=
+go.etcd.io/etcd/api/v3 v3.6.10 h1:jlwjtELjA8yi2VWpOFH+0w0lGr3K6mVDyn0RDB9aaAY=
+go.etcd.io/etcd/api/v3 v3.6.10/go.mod h1:pdV4VeFmvhdNjB4LWRkC8ReLyRBAxUOze3GarMhE2sk=
+go.etcd.io/etcd/client/pkg/v3 v3.6.10 h1:tBT7podcPhuVbCVkAEzx8bC5I+aqxfLwBN8/As1arrA=
+go.etcd.io/etcd/client/pkg/v3 v3.6.10/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
+go.etcd.io/etcd/client/v3 v3.6.10 h1:J598zJ+C/ZPvImypmq5waj84+bovePrlZERHklf34y0=
+go.etcd.io/etcd/client/v3 v3.6.10/go.mod h1:iHhUDUcEwaKs1YFq3MgmI9U4zhTVasp/vgdVbFf1RS8=
 go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
 go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -2130,8 +2128,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0/go.mod h1:CosX/aS4eHnG9D7nESYpV753l4j9q5j3SL/PUYd2lR8=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
-go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
-go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
+go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
+go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8=
@@ -2146,14 +2144,14 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
 go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
 go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
-go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
-go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
-go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
-go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
-go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
-go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
-go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
-go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
+go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
+go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
+go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
+go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
+go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
+go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
+go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
+go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
@@ -2242,8 +2240,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
 golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
-golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
-golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
+golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
+golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -2346,8 +2344,8 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
-golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2554,8 +2552,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
-golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -2705,8 +2703,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/
 google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
 google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
 google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
-google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
-google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
+google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
+google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -2840,12 +2838,12 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
 google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
 google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
 google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
-google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
-google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
-google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
-google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
+google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
+google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=
+google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.17"
+appVersion: "4.19"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.17.0
+version: 4.19.0

other/java/client/src/main/proto/filer.proto

@@ -84,6 +84,8 @@ service SeaweedFiler {
     // distributed lock management internal use only
     rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
     }
+    rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
+    }
 }
 
 //////////////////////////////////////////////////
@@ -522,6 +524,7 @@ message LockResponse {
     string lock_owner = 2;
     string lock_host_moved_to = 3;
     string error = 4;
+    int64 generation = 5;
 }
 message UnlockRequest {
     string name = 1;
@@ -544,12 +547,26 @@ message Lock {
     string renew_token = 2;
     int64 expired_at_ns = 3;
     string owner = 4;
+    int64 generation = 5;
+    bool is_backup = 6;
+    int64 seq = 7;
 }
 message TransferLocksRequest {
     repeated Lock locks = 1;
 }
 message TransferLocksResponse {
 }
+message ReplicateLockRequest {
+    string name = 1;
+    string renew_token = 2;
+    int64 expired_at_ns = 3;
+    string owner = 4;
+    int64 generation = 5;
+    bool is_unlock = 6;
+    int64 seq = 7;
+}
+message ReplicateLockResponse {
+}
 
 //////////////////////////////////////////////////
 // StreamMutateEntry: ordered bidirectional streaming for all filer mutations.

seaweed-volume/Cargo.toml

@@ -20,7 +20,7 @@ default = ["5bytes"]
 [dependencies]
 # Async runtime
 tokio = { version = "1", features = ["full"] }
-tokio-stream = "0.1"
+tokio-stream = { version = "0.1", features = ["net"] }
 tokio-io-timeout = "1"
 
 # gRPC + protobuf
@@ -65,8 +65,6 @@ reed-solomon-erasure = "6"
 # Logging
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-pprof = { version = "0.15", features = ["prost-codec"] }
-
 # Config
 toml = "0.8"
 serde = { version = "1", features = ["derive"] }
@@ -84,7 +82,7 @@ memmap2 = "0.9"
 uuid = { version = "1", features = ["v4"] }
 
 # HTTP client (for proxying, remote fetch)
-reqwest = { version = "0.12", features = ["rustls-tls", "stream", "multipart", "json"] }
+reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream", "multipart", "json"] }
 
 # Content hashing
 md-5 = "0.10"
@@ -127,6 +125,10 @@ aws-sdk-s3 = { version = "1.125.0", default-features = false, features = ["sigv4
 aws-credential-types = "1"
 aws-types = "1"
 
+# pprof is Unix-only (requires libc/nix APIs not available on Windows)
+[target.'cfg(unix)'.dependencies]
+pprof = { version = "0.15", features = ["prost-codec"] }
+
 [dev-dependencies]
 tempfile = "3"
 

seaweed-volume/src/main.rs

@@ -10,9 +10,11 @@ use seaweed_volume::security::tls::{
     GrpcClientAuthPolicy, TlsPolicy,
 };
 use seaweed_volume::security::{Guard, SigningKey};
+#[cfg(unix)]
 use seaweed_volume::server::debug::build_debug_router;
 use seaweed_volume::server::grpc_client::load_outgoing_grpc_tls;
 use seaweed_volume::server::grpc_server::VolumeGrpcService;
+#[cfg(unix)]
 use seaweed_volume::server::profiling::CpuProfileSession;
 use seaweed_volume::server::request_id::GrpcRequestIdLayer;
 use seaweed_volume::server::volume_server::{
@@ -24,6 +26,11 @@ use seaweed_volume::storage::types::DiskType;
 
 use tokio_rustls::TlsAcceptor;
 
+#[cfg(unix)]
+type CpuProfileParam = Option<CpuProfileSession>;
+#[cfg(not(unix))]
+type CpuProfileParam = Option<()>;
+
 const GRPC_MAX_MESSAGE_SIZE: usize = 1 << 30;
 const GRPC_KEEPALIVE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);
 const GRPC_KEEPALIVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
@@ -42,6 +49,7 @@ fn main() {
 
     let config = config::parse_cli();
     seaweed_volume::server::server_stats::init_process_start();
+    #[cfg(unix)]
     let cpu_profile = match CpuProfileSession::start(&config) {
         Ok(session) => session,
         Err(e) => {
@@ -49,6 +57,8 @@ fn main() {
             std::process::exit(1);
         }
     };
+    #[cfg(not(unix))]
+    let cpu_profile: Option<()> = None;
     info!(
         "SeaweedFS Volume Server (Rust) v{}",
         seaweed_volume::version::full_version()
@@ -257,7 +267,7 @@ where
 
 async fn run(
     config: VolumeServerConfig,
-    cpu_profile: Option<CpuProfileSession>,
+    #[allow(unused_variables)] cpu_profile: CpuProfileParam,
 ) -> Result<(), Box<dyn std::error::Error>> {
     // Initialize the store
     let mut store = Store::new(config.index_type);
@@ -431,10 +441,12 @@ async fn run(
     }
 
     // Build HTTP routers
+    #[allow(unused_mut)]
     let mut admin_router = seaweed_volume::server::volume_server::build_admin_router_with_ui(
         state.clone(),
         config.ui_enabled,
     );
+    #[cfg(unix)]
     if config.pprof {
         admin_router = admin_router.merge(build_debug_router());
     }
@@ -588,31 +600,35 @@ async fn run(
         })
     };
 
+    // Bind the gRPC listener before spawning to propagate bind errors at startup.
+    let grpc_listener = tokio::net::TcpListener::bind(&grpc_addr)
+        .await
+        .unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
+    let grpc_local_addr = grpc_listener
+        .local_addr()
+        .unwrap_or_else(|e| panic!("Failed to get gRPC local addr: {}", e));
+
     let grpc_handle = {
         let grpc_state = state.clone();
-        let grpc_addr = grpc_addr.clone();
         let grpc_tls_acceptor = grpc_tls_acceptor.clone();
         let mut shutdown_rx = shutdown_tx.subscribe();
+        let shutdown_tx_grpc = shutdown_tx.clone();
         tokio::spawn(async move {
-            let addr = grpc_addr.parse().expect("Invalid gRPC address");
             let grpc_service = VolumeGrpcService {
                 state: grpc_state.clone(),
             };
-            if let Some(tls_acceptor) = grpc_tls_acceptor {
-                let listener = tokio::net::TcpListener::bind(&grpc_addr)
-                    .await
-                    .unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
-                let incoming = grpc_tls_incoming(listener, tls_acceptor);
-                let reflection_v1 = tonic_reflection::server::Builder::configure()
-                    .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
-                    .build_v1()
-                    .expect("Failed to build gRPC reflection v1 service");
-                let reflection_v1alpha = tonic_reflection::server::Builder::configure()
-                    .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
-                    .build_v1alpha()
-                    .expect("Failed to build gRPC reflection v1alpha service");
-                info!("gRPC server listening on {} (TLS enabled)", addr);
-                if let Err(e) = build_grpc_server_builder()
+            let reflection_v1 = tonic_reflection::server::Builder::configure()
+                .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
+                .build_v1()
+                .expect("Failed to build gRPC reflection v1 service");
+            let reflection_v1alpha = tonic_reflection::server::Builder::configure()
+                .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
+                .build_v1alpha()
+                .expect("Failed to build gRPC reflection v1alpha service");
+            let result = if let Some(tls_acceptor) = grpc_tls_acceptor {
+                let incoming = grpc_tls_incoming(grpc_listener, tls_acceptor);
+                info!("gRPC server listening on {} (TLS enabled)", grpc_local_addr);
+                build_grpc_server_builder()
                     .layer(GrpcRequestIdLayer)
                     .add_service(reflection_v1)
                     .add_service(reflection_v1alpha)
@@ -621,32 +637,25 @@ async fn run(
                         let _ = shutdown_rx.recv().await;
                     })
                     .await
-                {
-                    error!("gRPC server error: {}", e);
-                }
             } else {
-                let reflection_v1 = tonic_reflection::server::Builder::configure()
-                    .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
-                    .build_v1()
-                    .expect("Failed to build gRPC reflection v1 service");
-                let reflection_v1alpha = tonic_reflection::server::Builder::configure()
-                    .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
-                    .build_v1alpha()
-                    .expect("Failed to build gRPC reflection v1alpha service");
-                info!("gRPC server listening on {}", addr);
-                if let Err(e) = build_grpc_server_builder()
+                let incoming =
+                    tokio_stream::wrappers::TcpListenerStream::new(grpc_listener);
+                info!("gRPC server listening on {}", grpc_local_addr);
+                build_grpc_server_builder()
                     .layer(GrpcRequestIdLayer)
                     .add_service(reflection_v1)
                     .add_service(reflection_v1alpha)
                     .add_service(build_volume_grpc_service(grpc_service))
-                    .serve_with_shutdown(addr, async move {
+                    .serve_with_incoming_shutdown(incoming, async move {
                         let _ = shutdown_rx.recv().await;
                     })
                     .await
-                {
-                    error!("gRPC server error: {}", e);
-                }
+            };
+            if let Err(ref e) = result {
+                error!("gRPC server error: {}", e);
+                let _ = shutdown_tx_grpc.send(());
             }
+            result
         })
     };
 
@@ -721,6 +730,7 @@ async fn run(
         None
     };
 
+    #[cfg(unix)]
     let debug_handle = if config.debug {
         let debug_addr = format!("0.0.0.0:{}", config.debug_port);
         info!("Debug pprof server listening on {}", debug_addr);
@@ -742,6 +752,8 @@ async fn run(
     } else {
         None
     };
+    #[cfg(not(unix))]
+    let debug_handle: Option<tokio::task::JoinHandle<()>> = None;
 
     let metrics_push_handle = {
         let push_state = state.clone();
@@ -752,9 +764,40 @@ async fn run(
         }))
     };
 
-    // Wait for all servers
+    // Wait for servers. Use select! with &mut so the losing handle is not
+    // dropped, then await it explicitly afterward.
+    let mut server_err: Option<String> = None;
+    let mut http_handle = http_handle;
+    let mut grpc_handle = grpc_handle;
+    let grpc_finished_first = tokio::select! {
+        _ = &mut http_handle => false,
+        _ = &mut grpc_handle => true,
+    };
+    // Inspect the gRPC result (already resolved if it finished first,
+    // otherwise await it now).
+    let grpc_result = if grpc_finished_first {
+        grpc_handle.await
+    } else {
+        // HTTP finished first; gRPC is still running. Await it.
+        grpc_handle.await
+    };
+    match grpc_result {
+        Ok(Ok(())) => {}
+        Ok(Err(e)) => {
+            let msg = format!("gRPC server exited with error: {}", e);
+            error!("{}", msg);
+            server_err = Some(msg);
+            // serve error already sent shutdown inside the task
+        }
+        Err(e) => {
+            let msg = format!("gRPC task panicked: {}", e);
+            error!("{}", msg);
+            server_err = Some(msg);
+            let _ = shutdown_tx.send(());
+        }
+    }
+    // Ensure the HTTP handle completes too.
     let _ = http_handle.await;
-    let _ = grpc_handle.await;
     if let Some(h) = public_handle {
         let _ = h.await;
     }
@@ -774,10 +817,15 @@ async fn run(
     // Close all volumes (flush and release file handles) matching Go's Shutdown()
     state.store.write().unwrap().close();
 
+    #[cfg(unix)]
     if let Some(cpu_profile) = cpu_profile {
         cpu_profile.finish().map_err(std::io::Error::other)?;
     }
 
+    if let Some(err_msg) = server_err {
+        return Err(std::io::Error::other(err_msg).into());
+    }
+
     info!("Volume server stopped.");
     Ok(())
 }

seaweed-volume/src/server/grpc_server.rs

@@ -1867,6 +1867,7 @@ impl VolumeServer for VolumeGrpcService {
         {
             let needle_header = resp.needle_header;
             let mut needle_body = resp.needle_body;
+            let resp_version = resp.version;
 
             if needle_header.is_empty() {
                 continue;
@@ -1891,8 +1892,36 @@ impl VolumeServer for VolumeGrpcService {
             // Parse needle from header + body
             let mut n = Needle::default();
             n.read_header(&needle_header);
-            n.read_body_v2(&needle_body)
-                .map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
+
+            if n.size.0 < 0 {
+                return Err(Status::invalid_argument(format!(
+                    "unexpected negative needle size {} for needle {}",
+                    n.size.0, n.id.0
+                )));
+            } else if n.size.0 > 0 {
+                // Normal needle: parse the body fields (DataSize, Data, flags, etc.)
+                n.read_body_v2(&needle_body)
+                    .map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
+            } else {
+                // Delete tombstone (size == 0): body is checksum + timestamp
+                // (V3) or checksum only (V2) + padding. Validate minimum
+                // footer length for the protocol version.
+                use crate::storage::types::{
+                    NEEDLE_CHECKSUM_SIZE, TIMESTAMP_SIZE, VERSION_3, Version,
+                };
+                let version = Version(resp_version as u8);
+                let min_footer = if version >= VERSION_3 {
+                    NEEDLE_CHECKSUM_SIZE + TIMESTAMP_SIZE
+                } else {
+                    NEEDLE_CHECKSUM_SIZE
+                };
+                if needle_body.len() < min_footer {
+                    return Err(Status::invalid_argument(format!(
+                        "tombstone needle {} body too short: got {} bytes, need >= {} for version {}",
+                        n.id.0, needle_body.len(), min_footer, resp_version
+                    )));
+                }
+            }
 
             // Write needle to local volume
             let mut store = state.store.write().unwrap();
@@ -4047,11 +4076,12 @@ fn find_last_append_at_ns(idx_path: &str, dat_path: &str, version: u32) -> Optio
     let mut header = [0u8; 16];
     dat_file.read_exact(&mut header).ok()?;
     let needle_size = i32::from_be_bytes([header[12], header[13], header[14], header[15]]);
-    if needle_size <= 0 {
+    if needle_size < 0 {
         return None;
     }
 
     // Seek to tail: offset + 16 (header) + size -> checksum (4) + timestamp (8)
+    // For delete needles (size == 0), the tail is right after the header.
     let tail_offset = actual_offset as u64 + 16 + needle_size as u64;
     dat_file.seek(SeekFrom::Start(tail_offset)).ok()?;
 

seaweed-volume/src/server/mod.rs

@@ -1,9 +1,11 @@
+#[cfg(unix)]
 pub mod debug;
 pub mod grpc_client;
 pub mod grpc_server;
 pub mod handlers;
 pub mod heartbeat;
 pub mod memory_status;
+#[cfg(unix)]
 pub mod profiling;
 pub mod request_id;
 pub mod server_stats;

seaweed-volume/src/storage/erasure_coding/ec_encoder.rs

@@ -6,7 +6,7 @@
 use std::fs::File;
 use std::io;
 #[cfg(not(unix))]
-use std::io::{Seek, SeekFrom};
+use std::io::{Read, Seek, SeekFrom};
 
 use reed_solomon_erasure::galois_8::ReedSolomon;
 

test/fuse_integration/concurrent_operations_test.go

@@ -249,11 +249,18 @@ func testConcurrentDirectoryOperations(t *testing.T, framework *FuseTestFramewor
 					return
 				}
 
-				// Create file in subdirectory
+				// Create file in subdirectory with retry for transient FUSE errors
 				testFile := filepath.Join(subDir, "test.txt")
 				content := []byte(fmt.Sprintf("Worker %d, Subdir %d", workerID, i))
-				if err := os.WriteFile(testFile, content, 0644); err != nil {
-					addError(fmt.Errorf("worker %d file %d: %v", workerID, i, err))
+				var writeErr error
+				for attempt := 0; attempt < 3; attempt++ {
+					if writeErr = os.WriteFile(testFile, content, 0644); writeErr == nil {
+						break
+					}
+					time.Sleep(100 * time.Millisecond)
+				}
+				if writeErr != nil {
+					addError(fmt.Errorf("worker %d file %d: %v", workerID, i, writeErr))
 					return
 				}
 			}

test/fuse_integration/git_operations_test.go

@@ -85,9 +85,19 @@ func testGitCloneAndPull(t *testing.T, mountPoint, localDir string) {
 	branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
 	gitRun(t, localClone, "push", "origin", branch)
 
+	// The bare repo lives on the FUSE mount and can briefly disappear after
+	// the push completes, and pushed pack objects may not be immediately
+	// consistent on the FUSE layer. Give the mount a chance to settle, then
+	// recover from the local clone if the remote is still missing.
+	if !waitForBareRepoEventually(t, bareRepo, 10*time.Second) {
+		t.Logf("bare repo %s did not stabilise after push; forcing recovery before clone", bareRepo)
+	}
+	refreshDirEntry(t, bareRepo)
+	time.Sleep(1 * time.Second)
+
 	// ---- Phase 3: Clone from mount bare repo into on-mount working dir ----
 	t.Log("Phase 3: clone from mount bare repo to on-mount working dir")
-	gitRun(t, "", "clone", bareRepo, mountClone)
+	ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)
 
 	assertFileContains(t, filepath.Join(mountClone, "README.md"), "# Updated")
 	assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v2")
@@ -290,7 +300,7 @@ func waitForBareRepoEventually(t *testing.T, bareRepo string, timeout time.Durat
 	t.Helper()
 	deadline := time.Now().Add(timeout)
 	for time.Now().Before(deadline) {
-		if isBareRepo(bareRepo) {
+		if isBareRepoAccessible(bareRepo) {
 			return true
 		}
 		refreshDirEntry(t, bareRepo)
@@ -312,6 +322,14 @@ func isBareRepo(bareRepo string) bool {
 	return true
 }
 
+func isBareRepoAccessible(bareRepo string) bool {
+	if !isBareRepo(bareRepo) {
+		return false
+	}
+	out, err := tryGitCommand("", "--git-dir="+bareRepo, "rev-parse", "--is-bare-repository")
+	return err == nil && out == "true"
+}
+
 func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
 	t.Helper()
 	require.NoError(t, tryEnsureMountClone(bareRepo, mountClone))
@@ -320,7 +338,7 @@ func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
 // tryEnsureBareRepo verifies the bare repo on the FUSE mount exists.
 // If it has vanished, it re-creates it from the local clone.
 func tryEnsureBareRepo(bareRepo, localClone string) error {
-	if _, err := os.Stat(filepath.Join(bareRepo, "HEAD")); err == nil {
+	if isBareRepoAccessible(bareRepo) {
 		return nil
 	}
 	branch, err := tryGitCommand(localClone, "rev-parse", "--abbrev-ref", "HEAD")
@@ -342,6 +360,36 @@ func tryEnsureBareRepo(bareRepo, localClone string) error {
 	return nil
 }
 
+func ensureMountCloneFromBareWithRecovery(t *testing.T, bareRepo, localClone, mountClone string) {
+	t.Helper()
+	const maxAttempts = 3
+	var lastErr error
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		if lastErr = tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone); lastErr == nil {
+			return
+		}
+		if attempt == maxAttempts {
+			require.NoError(t, lastErr, "git clone %s %s failed after %d recovery attempts", bareRepo, mountClone, maxAttempts)
+		}
+		t.Logf("clone recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
+		os.RemoveAll(mountClone)
+		time.Sleep(2 * time.Second)
+	}
+}
+
+func tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone string) error {
+	if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
+		return fmt.Errorf("ensure bare repo: %w", err)
+	}
+	if err := tryEnsureMountClone(bareRepo, mountClone); err != nil {
+		return fmt.Errorf("ensure mount clone: %w", err)
+	}
+	if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
+		return fmt.Errorf("verify mount clone: %w", err)
+	}
+	return nil
+}
+
 // tryEnsureMountClone is like ensureMountClone but returns an error instead
 // of failing the test, for use in recovery loops.
 func tryEnsureMountClone(bareRepo, mountClone string) error {
@@ -442,6 +490,20 @@ func resetToCommitWithRecovery(t *testing.T, bareRepo, localClone, mountClone, c
 			}
 			continue
 		}
+		// The kernel dcache can drop the FUSE entry moments after a
+		// successful check.  Wait briefly and re-verify to confirm the
+		// directory has stabilised before returning to the caller.
+		time.Sleep(1 * time.Second)
+		refreshDirEntry(t, mountClone)
+		if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
+			lastErr = fmt.Errorf("post-reset stabilisation check failed: %w", err)
+			if attempt < maxAttempts {
+				t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
+				os.RemoveAll(mountClone)
+				time.Sleep(2 * time.Second)
+			}
+			continue
+		}
 		return
 	}
 	require.NoError(t, lastErr, "git reset --hard %s failed after %d recovery attempts", commit, maxAttempts)
@@ -469,6 +531,10 @@ func tryPullFromCommit(t *testing.T, bareRepo, localClone, cloneDir, fromCommit
 	}
 	refreshDirEntry(t, cloneDir)
 
+	// Let the dcache stabilise before proceeding.
+	time.Sleep(1 * time.Second)
+	refreshDirEntry(t, cloneDir)
+
 	head, err := tryGitCommand(cloneDir, "rev-parse", "HEAD")
 	if err != nil {
 		return fmt.Errorf("rev-parse after reset: %w", err)
@@ -532,3 +598,33 @@ func TestTryEnsureBareRepoPreservesCurrentBranch(t *testing.T) {
 	assert.Equal(t, branch, restoredHead, "clone from recovered bare repo should check out the current branch")
 	assertFileContains(t, filepath.Join(restoredClone, "README.md"), "hello recovery")
 }
+
+func TestEnsureMountCloneFromBareWithRecoveryRecreatesMissingBareRepo(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "git_mount_clone_recovery_")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempDir)
+
+	bareRepo := filepath.Join(tempDir, "repo.git")
+	localClone := filepath.Join(tempDir, "clone")
+	mountClone := filepath.Join(tempDir, "mount-clone")
+
+	gitRun(t, "", "init", "--bare", bareRepo)
+	gitRun(t, "", "clone", bareRepo, localClone)
+	gitRun(t, localClone, "config", "user.email", "test@seaweedfs.test")
+	gitRun(t, localClone, "config", "user.name", "Test")
+
+	writeFile(t, localClone, "README.md", "hello clone recovery\n")
+	gitRun(t, localClone, "add", "README.md")
+	gitRun(t, localClone, "commit", "-m", "initial commit")
+
+	branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
+	gitRun(t, localClone, "push", "origin", branch)
+
+	require.NoError(t, os.RemoveAll(bareRepo))
+
+	ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)
+
+	head := gitOutput(t, mountClone, "rev-parse", "--abbrev-ref", "HEAD")
+	assert.Equal(t, branch, head, "recovered clone should stay on the pushed branch")
+	assertFileContains(t, filepath.Join(mountClone, "README.md"), "hello clone recovery")
+}

test/kafka/go.mod

@@ -14,7 +14,7 @@ require (
 replace github.com/seaweedfs/seaweedfs => ../../
 
 require (
-	cloud.google.com/go/auth v0.17.0 // indirect
+	cloud.google.com/go/auth v0.18.1 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
@@ -43,25 +43,25 @@ require (
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/aws/aws-sdk-go v1.55.8 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.41.4 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.32.7 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.32.13 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.13 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect
 	github.com/aws/smithy-go v1.24.2 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -104,7 +104,7 @@ require (
 	github.com/go-chi/chi/v5 v5.2.5 // indirect
 	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
 	github.com/go-git/go-billy/v5 v5.6.2 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
@@ -124,8 +124,8 @@ require (
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
-	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
+	github.com/googleapis/gax-go/v2 v2.17.0 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gorilla/schema v1.4.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -147,9 +147,9 @@ require (
 	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
 	github.com/karlseguin/ccache/v2 v2.0.8 // indirect
-	github.com/klauspost/compress v1.18.4 // indirect
+	github.com/klauspost/compress v1.18.5 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
-	github.com/klauspost/reedsolomon v1.13.0 // indirect
+	github.com/klauspost/reedsolomon v1.13.3 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
@@ -237,7 +237,7 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
-	github.com/zeebo/xxh3 v1.0.2 // indirect
+	github.com/zeebo/xxh3 v1.1.0 // indirect
 	go.etcd.io/bbolt v1.4.3 // indirect
 	go.mongodb.org/mongo-driver v1.17.9 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
@@ -247,18 +247,18 @@ require (
 	go.opentelemetry.io/otel/trace v1.40.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.48.0 // indirect
-	golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
-	golang.org/x/image v0.36.0 // indirect
-	golang.org/x/net v0.49.0 // indirect
+	golang.org/x/crypto v0.49.0 // indirect
+	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
+	golang.org/x/image v0.38.0 // indirect
+	golang.org/x/net v0.51.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.42.0 // indirect
-	golang.org/x/term v0.40.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
+	golang.org/x/term v0.41.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
-	google.golang.org/api v0.258.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
+	google.golang.org/api v0.267.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
 	google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect

test/kafka/go.sum

@@ -13,8 +13,8 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
 cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4=
-cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ=
+cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
+cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
@@ -112,44 +112,44 @@ github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc h1:LoL75er
 github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc/go.mod h1:w648aMHEgFYS6xb0KVMMtZ2uMeemhiKCuD2vj6gY52A=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
-github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
+github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
+github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
-github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
+github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
+github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZifkk6vEGl/1Qor0=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
 github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
 github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
@@ -284,8 +284,8 @@ github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
-github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
+github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -385,12 +385,12 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
 github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ=
-github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
+github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
+github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
-github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
+github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
+github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
 github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
 github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
@@ -467,12 +467,12 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt
 github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
-github.com/klauspost/reedsolomon v1.13.0 h1:E0Cmgf2kMuhZTj6eefnvpKC4/Q4jhCi9YIjcZjK4arc=
-github.com/klauspost/reedsolomon v1.13.0/go.mod h1:ggJT9lc71Vu+cSOPBlxGvBN6TfAS77qB4fp8vJ05NSA=
+github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
+github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 h1:CjEMN21Xkr9+zwPmZPaJJw+apzVbjGL5uK/6g9Q2jGU=
 github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988/go.mod h1:/agobYum3uo/8V6yPVnq+R82pyVGCeuWW5arT4Txn8A=
 github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 h1:FHVoZMOVRA+6/y4yRlbiR3WvsrOcKBd/f64H7YiWR2U=
@@ -707,8 +707,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
-github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
+github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -739,8 +739,8 @@ github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
 github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
 github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
 github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
-github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
-github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
+github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
+github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
 go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
 go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
@@ -787,8 +787,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
-golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
+golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -799,12 +799,12 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
-golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
+golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
+golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
-golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
+golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
+golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -869,8 +869,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
-golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
+golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
+golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -894,8 +894,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -959,8 +959,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
-golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
-golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
+golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
+golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -976,8 +976,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1030,14 +1030,14 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
-golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
+golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
+golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
-gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -1054,8 +1054,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.258.0 h1:IKo1j5FBlN74fe5isA2PVozN3Y5pwNKriEgAXPOkDAc=
-google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww=
+google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
+google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1091,12 +1091,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 h1:dDbsTLIK7EzwUq36kCSAsk0slouq/S0tWHeeGi97cD8=
-google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846/go.mod h1:PP0g88Dz3C7hRAfbQCQggeWAXjuqGsNPLE4s7jh0RGU=
-google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
-google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
+google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
+google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
+google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
+google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

test/plugin_workers/fake_volume_server.go

@@ -11,9 +11,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
 // VolumeServer provides a minimal volume server for erasure coding tests.
@@ -196,12 +198,25 @@ func (v *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream vo
 	defer file.Close()
 
 	buf := make([]byte, 64*1024)
+	remaining := int64(req.GetStopOffset())
 	for {
-		n, readErr := file.Read(buf)
+		if remaining == 0 {
+			break
+		}
+
+		readBuf := buf
+		if remaining > 0 && remaining < int64(len(buf)) {
+			readBuf = buf[:remaining]
+		}
+
+		n, readErr := file.Read(readBuf)
 		if n > 0 {
-			if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: buf[:n]}); err != nil {
+			if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: readBuf[:n]}); err != nil {
 				return err
 			}
+			if remaining > 0 {
+				remaining -= int64(n)
+			}
 		}
 		if readErr == io.EOF {
 			break
@@ -307,10 +322,21 @@ func (v *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_ser
 	v.mu.Lock()
 	v.readFileStatusCalls++
 	v.mu.Unlock()
+
+	datInfo, err := os.Stat(v.filePath(req.VolumeId, ".dat"))
+	if err != nil {
+		return nil, err
+	}
+
+	idxInfo, err := os.Stat(v.filePath(req.VolumeId, ".idx"))
+	if err != nil {
+		return nil, err
+	}
+
 	return &volume_server_pb.ReadVolumeFileStatusResponse{
 		VolumeId:    req.VolumeId,
-		DatFileSize: 1024,
-		IdxFileSize: 16,
+		DatFileSize: uint64(datInfo.Size()),
+		IdxFileSize: uint64(idxInfo.Size()),
 		FileCount:   1,
 	}, nil
 }
@@ -349,7 +375,27 @@ func (v *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, strea
 	v.volumeCopyCalls++
 	v.mu.Unlock()
 
-	if err := stream.Send(&volume_server_pb.VolumeCopyResponse{ProcessedBytes: 1024}); err != nil {
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	var statusResp *volume_server_pb.ReadVolumeFileStatusResponse
+	if err := operation.WithVolumeServerClient(false, pb.ServerAddress(req.SourceDataNode), dialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			var readErr error
+			statusResp, readErr = client.ReadVolumeFileStatus(stream.Context(), &volume_server_pb.ReadVolumeFileStatusRequest{
+				VolumeId: req.VolumeId,
+			})
+			return readErr
+		}); err != nil {
+		return err
+	}
+
+	if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".dat", statusResp.DatFileSize, dialOption); err != nil {
+		return err
+	}
+	if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".idx", statusResp.IdxFileSize, dialOption); err != nil {
+		return err
+	}
+
+	if err := stream.Send(&volume_server_pb.VolumeCopyResponse{ProcessedBytes: int64(statusResp.DatFileSize + statusResp.IdxFileSize)}); err != nil {
 		return err
 	}
 	return stream.Send(&volume_server_pb.VolumeCopyResponse{LastAppendAtNs: uint64(time.Now().UnixNano())})
@@ -368,3 +414,44 @@ func (v *VolumeServer) VolumeTailReceiver(ctx context.Context, req *volume_serve
 	v.mu.Unlock()
 	return &volume_server_pb.VolumeTailReceiverResponse{}, nil
 }
+
+func (v *VolumeServer) copyRemoteFile(ctx context.Context, sourceDataNode string, volumeID uint32, ext string, fileSize uint64, dialOption grpc.DialOption) error {
+	path := v.filePath(volumeID, ext)
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return err
+	}
+
+	file, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	return operation.WithVolumeServerClient(true, pb.ServerAddress(sourceDataNode), dialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			stream, err := client.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+				VolumeId:   volumeID,
+				Ext:        ext,
+				StopOffset: fileSize,
+			})
+			if err != nil {
+				return err
+			}
+
+			for {
+				resp, recvErr := stream.Recv()
+				if recvErr == io.EOF {
+					return nil
+				}
+				if recvErr != nil {
+					return recvErr
+				}
+				if len(resp.FileContent) == 0 {
+					continue
+				}
+				if _, err := file.Write(resp.FileContent); err != nil {
+					return err
+				}
+			}
+		})
+}

test/plugin_workers/volume_balance/execution_test.go

@@ -16,6 +16,8 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+const testVolumeDatSize = 1 * 1024 * 1024
+
 func TestVolumeBalanceExecutionIntegration(t *testing.T) {
 	volumeID := uint32(303)
 
@@ -31,6 +33,7 @@ func TestVolumeBalanceExecutionIntegration(t *testing.T) {
 
 	source := pluginworkers.NewVolumeServer(t, "")
 	target := pluginworkers.NewVolumeServer(t, "")
+	pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), volumeID, testVolumeDatSize)
 
 	job := &plugin_pb.JobSpec{
 		JobId:   fmt.Sprintf("balance-job-%d", volumeID),
@@ -84,6 +87,9 @@ func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
 
 	// Build a batch job with 3 volume moves from source → target.
 	volumeIDs := []uint32{401, 402, 403}
+	for _, vid := range volumeIDs {
+		pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), vid, testVolumeDatSize)
+	}
 	moves := make([]*worker_pb.BalanceMoveSpec, len(volumeIDs))
 	for i, vid := range volumeIDs {
 		moves[i] = &worker_pb.BalanceMoveSpec{
@@ -139,10 +145,11 @@ func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
 		require.True(t, deletedVols[vid], "volume %d should have been deleted from source", vid)
 	}
 
-	// Pre-delete verification should have called ReadVolumeFileStatus on both
-	// source and target for each volume.
-	require.Equal(t, len(volumeIDs), source.ReadFileStatusCount(),
-		"each move should read source volume status before delete")
+	// Each move reads source status once before copy and once inside the
+	// target's fake VolumeCopy implementation, then reads target status once
+	// before deleting the source.
+	require.Equal(t, len(volumeIDs)*2, source.ReadFileStatusCount(),
+		"each move should read source volume status before copy and during target copy")
 	require.Equal(t, len(volumeIDs), target.ReadFileStatusCount(),
 		"each move should read target volume status before delete")
 

test/plugin_workers/volume_fixtures.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 )
 
@@ -21,7 +22,12 @@ func WriteTestVolumeFiles(t *testing.T, baseDir string, volumeID uint32, datSize
 	datPath := filepath.Join(baseDir, volumeFilename(volumeID, ".dat"))
 	idxPath := filepath.Join(baseDir, volumeFilename(volumeID, ".idx"))
 
-	data := make([]byte, datSize)
+	// The idx entry records the needle's data size. The actual on-disk size
+	// includes header + checksum + timestamp (GetActualSize). The .dat must
+	// be large enough to hold the full needle.
+	needleDataSize := types.Size(datSize)
+	actualSize := needle.GetActualSize(needleDataSize, needle.Version3)
+	data := make([]byte, actualSize)
 	rng := rand.New(rand.NewSource(99))
 	_, _ = rng.Read(data)
 	if err := os.WriteFile(datPath, data, 0644); err != nil {
@@ -35,7 +41,7 @@ func WriteTestVolumeFiles(t *testing.T, baseDir string, volumeID uint32, datSize
 
 	types.NeedleIdToBytes(entry[:idEnd], types.NeedleId(1))
 	types.OffsetToBytes(entry[idEnd:offsetEnd], types.ToOffset(0))
-	types.SizeToBytes(entry[offsetEnd:sizeEnd], types.Size(datSize))
+	types.SizeToBytes(entry[offsetEnd:sizeEnd], needleDataSize)
 
 	if err := os.WriteFile(idxPath, entry, 0644); err != nil {
 		t.Fatalf("write idx file: %v", err)

test/s3/distributed_lock/distributed_lock_cluster_test.go

@@ -21,7 +21,9 @@ import (
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc"
@@ -139,6 +141,7 @@ func startDistributedLockCluster(t *testing.T) *distributedLockCluster {
 		require.NoError(t, cluster.waitForTCP(cluster.filerGRPCAddress(i), 30*time.Second), "wait for filer %d grpc\n%s", i, cluster.tailLog(fmt.Sprintf("filer%d.log", i)))
 	}
 	require.NoError(t, cluster.waitForFilerCount(2, 30*time.Second), "wait for filer group registration")
+	require.NoError(t, cluster.waitForLockRingConverged(30*time.Second), "wait for lock ring convergence")
 
 	for i := 0; i < 2; i++ {
 		require.NoError(t, cluster.startS3(i), "start s3 %d", i)
@@ -411,6 +414,130 @@ func (c *distributedLockCluster) waitForFilerCount(expected int, timeout time.Du
 	return fmt.Errorf("timed out waiting for %d filers in group %q", expected, c.filerGroup)
 }
 
+// waitForLockRingConverged verifies that both filers have a consistent view of the
+// lock ring by acquiring the same lock through each filer and checking mutual exclusion.
+// This guards against the window between master seeing both filers and the filers
+// actually receiving the LockRingUpdate broadcast (delayed by the stabilization timer).
+//
+// A single arbitrary key could false-pass: if the key's real primary is filer0 and
+// filer0 has a stale ring (only sees itself), it still grants correctly because it IS
+// the primary. To catch the stale ring we must also test a key whose real primary is
+// filer1. So we generate one test key per primary filer using the same consistent-hash
+// ring as production, and require mutual exclusion for all of them.
+func (c *distributedLockCluster) waitForLockRingConverged(timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+
+	owners := make([]pb.ServerAddress, 0, len(c.filerPorts))
+	for i := range c.filerPorts {
+		owners = append(owners, c.filerServerAddress(i))
+	}
+	ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	ring.SetServers(owners)
+
+	attempt := 0
+	for time.Now().Before(deadline) {
+		// Generate one unique test key per primary filer
+		testKeys := c.convergenceKeysPerPrimary(ring, owners, attempt)
+		attempt++
+
+		allConverged := true
+		for _, key := range testKeys {
+			converged, _ := c.checkLockMutualExclusion(key)
+			if !converged {
+				allConverged = false
+				break
+			}
+		}
+		if allConverged {
+			return nil
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("lock ring did not converge: both filers independently grant the same lock")
+}
+
+// convergenceKeysPerPrimary returns one lock key per distinct primary filer,
+// using the same consistent-hash ring as production routing.
+func (c *distributedLockCluster) convergenceKeysPerPrimary(ring *lock_manager.HashRing, owners []pb.ServerAddress, attempt int) []string {
+	keysByOwner := make(map[pb.ServerAddress]string, len(owners))
+	for i := 0; len(keysByOwner) < len(owners) && i < 1024; i++ {
+		candidate := fmt.Sprintf("convergence-%d-%d", attempt, i)
+		primary := ring.GetPrimary(candidate)
+		if _, exists := keysByOwner[primary]; !exists {
+			keysByOwner[primary] = candidate
+		}
+	}
+	keys := make([]string, 0, len(keysByOwner))
+	for _, k := range keysByOwner {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+// checkLockMutualExclusion acquires a lock via filer0, then tries the same lock via filer1.
+// Returns true if mutual exclusion holds (second attempt is denied).
+func (c *distributedLockCluster) checkLockMutualExclusion(testKey string) (bool, error) {
+	conn0, err := grpc.NewClient(c.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false, err
+	}
+	defer conn0.Close()
+
+	conn1, err := grpc.NewClient(c.filerGRPCAddress(1), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false, err
+	}
+	defer conn1.Close()
+
+	client0 := filer_pb.NewSeaweedFilerClient(conn0)
+	client1 := filer_pb.NewSeaweedFilerClient(conn1)
+
+	// Acquire lock via filer0
+	ctx0, cancel0 := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel0()
+	resp0, err := client0.DistributedLock(ctx0, &filer_pb.LockRequest{
+		Name:          testKey,
+		SecondsToLock: 5,
+		Owner:         "convergence-filer0",
+	})
+	if err != nil || resp0.RenewToken == "" {
+		return false, fmt.Errorf("filer0 lock failed: err=%v resp=%v", err, resp0)
+	}
+	defer func() {
+		// Always release the lock we acquired
+		client0.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
+			Name:       testKey,
+			RenewToken: resp0.RenewToken,
+		})
+	}()
+
+	// Try the same lock via filer1 - should be denied
+	ctx1, cancel1 := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel1()
+	resp1, err := client1.DistributedLock(ctx1, &filer_pb.LockRequest{
+		Name:          testKey,
+		SecondsToLock: 5,
+		Owner:         "convergence-filer1",
+	})
+	if err != nil {
+		return false, err
+	}
+	if resp1.RenewToken != "" {
+		// Both filers granted the lock - ring not converged. Release the second lock.
+		client1.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
+			Name:       testKey,
+			RenewToken: resp1.RenewToken,
+		})
+		return false, nil
+	}
+	// Verify the denial is specifically because the lock is already held,
+	// not due to a transient error that might give a false positive.
+	if !strings.Contains(resp1.Error, "lock already owned") {
+		return false, nil
+	}
+	return true, nil
+}
+
 func (c *distributedLockCluster) waitForHTTP(url string, timeout time.Duration) error {
 	client := &net.Dialer{Timeout: time.Second}
 	httpClient := &httpClientWithDialer{dialer: client}

test/s3/distributed_lock/distributed_lock_test.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"sort"
 	"strings"
 	"sync"
 	"testing"
@@ -15,9 +14,9 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/smithy-go"
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -149,14 +148,15 @@ func (c *distributedLockCluster) findLockOwnerKeys(bucket, prefix string) map[pb
 	for i := range c.filerPorts {
 		owners = append(owners, c.filerServerAddress(i))
 	}
-	sort.Slice(owners, func(i, j int) bool {
-		return owners[i] < owners[j]
-	})
+
+	ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	ring.SetServers(owners)
 
 	keysByOwner := make(map[pb.ServerAddress]string, len(owners))
 	for i := 0; i < 1024 && len(keysByOwner) < len(owners); i++ {
 		key := fmt.Sprintf("%s-%03d.txt", prefix, i)
-		lockOwner := ownerForObjectLock(bucket, key, owners)
+		lockKey := fmt.Sprintf("s3.object.write:/buckets/%s/%s", bucket, s3_constants.NormalizeObjectKey(key))
+		lockOwner := ring.GetPrimary(lockKey)
 		if _, exists := keysByOwner[lockOwner]; !exists {
 			keysByOwner[lockOwner] = key
 		}
@@ -164,15 +164,6 @@ func (c *distributedLockCluster) findLockOwnerKeys(bucket, prefix string) map[pb
 	return keysByOwner
 }
 
-func ownerForObjectLock(bucket, object string, owners []pb.ServerAddress) pb.ServerAddress {
-	lockKey := fmt.Sprintf("s3.object.write:/buckets/%s/%s", bucket, s3_constants.NormalizeObjectKey(object))
-	hash := util.HashStringToLong(lockKey)
-	if hash < 0 {
-		hash = -hash
-	}
-	return owners[hash%int64(len(owners))]
-}
-
 func lockOwnerLabel(owner pb.ServerAddress) string {
 	replacer := strings.NewReplacer(":", "_", ".", "_")
 	return "owner_" + replacer.Replace(string(owner))

test/s3/iam/s3_sts_get_federation_token_test.go

@@ -0,0 +1,511 @@
+package iam
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// GetFederationTokenTestResponse represents the STS GetFederationToken response
+type GetFederationTokenTestResponse struct {
+	XMLName xml.Name `xml:"GetFederationTokenResponse"`
+	Result  struct {
+		Credentials struct {
+			AccessKeyId     string `xml:"AccessKeyId"`
+			SecretAccessKey string `xml:"SecretAccessKey"`
+			SessionToken    string `xml:"SessionToken"`
+			Expiration      string `xml:"Expiration"`
+		} `xml:"Credentials"`
+		FederatedUser struct {
+			FederatedUserId string `xml:"FederatedUserId"`
+			Arn             string `xml:"Arn"`
+		} `xml:"FederatedUser"`
+	} `xml:"GetFederationTokenResult"`
+}
+
+func getTestCredentials() (string, string) {
+	accessKey := os.Getenv("STS_TEST_ACCESS_KEY")
+	if accessKey == "" {
+		accessKey = "admin"
+	}
+	secretKey := os.Getenv("STS_TEST_SECRET_KEY")
+	if secretKey == "" {
+		secretKey = "admin"
+	}
+	return accessKey, secretKey
+}
+
+// isGetFederationTokenImplemented checks if the running server supports GetFederationToken
+func isGetFederationTokenImplemented(t *testing.T) bool {
+	accessKey, secretKey := getTestCredentials()
+	resp, err := callSTSAPIWithSigV4(t, url.Values{
+		"Action":  {"GetFederationToken"},
+		"Version": {"2011-06-15"},
+		"Name":    {"probe"},
+	}, accessKey, secretKey)
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+	var errResp STSErrorTestResponse
+	if xml.Unmarshal(body, &errResp) == nil {
+		if errResp.Error.Code == "InvalidAction" || errResp.Error.Code == "NotImplemented" {
+			return false
+		}
+	}
+	return true
+}
+
+// TestSTSGetFederationTokenValidation tests input validation for the GetFederationToken endpoint
+func TestSTSGetFederationTokenValidation(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Fatal("SeaweedFS STS endpoint is not running at", TestSTSEndpoint, "- please run 'make setup-all-tests' first")
+	}
+
+	if !isGetFederationTokenImplemented(t) {
+		t.Fatal("GetFederationToken action is not implemented in the running server")
+	}
+
+	accessKey, secretKey := getTestCredentials()
+
+	t.Run("missing_name", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			// Name is missing
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "MissingParameter", errResp.Error.Code)
+	})
+
+	t.Run("name_too_short", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"A"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("name_too_long", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {strings.Repeat("A", 33)},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("name_invalid_characters", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"bad name"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("duration_too_short", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"GetFederationToken"},
+			"Version":         {"2011-06-15"},
+			"Name":            {"TestApp"},
+			"DurationSeconds": {"100"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("duration_too_long", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"GetFederationToken"},
+			"Version":         {"2011-06-15"},
+			"Name":            {"TestApp"},
+			"DurationSeconds": {"200000"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("malformed_policy", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"TestApp"},
+			"Policy":  {"not-valid-json"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "MalformedPolicyDocument", errResp.Error.Code)
+	})
+
+	t.Run("anonymous_rejected", func(t *testing.T) {
+		// GetFederationToken requires SigV4, anonymous should fail
+		resp, err := callSTSAPI(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"TestApp"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode)
+	})
+}
+
+// TestSTSGetFederationTokenRejectTemporaryCredentials tests that temporary
+// credentials (session tokens) are rejected by GetFederationToken
+func TestSTSGetFederationTokenRejectTemporaryCredentials(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	if !isGetFederationTokenImplemented(t) {
+		t.Skip("GetFederationToken not implemented")
+	}
+
+	accessKey, secretKey := getTestCredentials()
+
+	// First, obtain temporary credentials via AssumeRole
+	resp, err := callSTSAPIWithSigV4(t, url.Values{
+		"Action":          {"AssumeRole"},
+		"Version":         {"2011-06-15"},
+		"RoleArn":         {"arn:aws:iam::role/admin"},
+		"RoleSessionName": {"temp-session"},
+	}, accessKey, secretKey)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	if resp.StatusCode != http.StatusOK {
+		t.Skipf("AssumeRole failed (may not be configured): status=%d body=%s", resp.StatusCode, string(body))
+	}
+
+	var assumeResp AssumeRoleTestResponse
+	require.NoError(t, xml.Unmarshal(body, &assumeResp), "Parse AssumeRole response: %s", string(body))
+
+	tempAccessKey := assumeResp.Result.Credentials.AccessKeyId
+	tempSecretKey := assumeResp.Result.Credentials.SecretAccessKey
+	tempSessionToken := assumeResp.Result.Credentials.SessionToken
+	require.NotEmpty(t, tempAccessKey)
+	require.NotEmpty(t, tempSessionToken)
+
+	// Now try GetFederationToken with the temporary credentials
+	// Include X-Amz-Security-Token header which marks this as a temp credential call
+	params := url.Values{
+		"Action":  {"GetFederationToken"},
+		"Version": {"2011-06-15"},
+		"Name":    {"ShouldFail"},
+	}
+
+	reqBody := params.Encode()
+	req, err := http.NewRequest(http.MethodPost, TestSTSEndpoint+"/", strings.NewReader(reqBody))
+	require.NoError(t, err)
+
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("X-Amz-Security-Token", tempSessionToken)
+
+	creds := credentials.NewStaticCredentials(tempAccessKey, tempSecretKey, tempSessionToken)
+	signer := v4.NewSigner(creds)
+	_, err = signer.Sign(req, strings.NewReader(reqBody), "sts", "us-east-1", time.Now())
+	require.NoError(t, err)
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	resp2, err := client.Do(req)
+	require.NoError(t, err)
+	defer resp2.Body.Close()
+
+	body2, _ := io.ReadAll(resp2.Body)
+	assert.Equal(t, http.StatusForbidden, resp2.StatusCode,
+		"GetFederationToken should reject temporary credentials: %s", string(body2))
+	assert.Contains(t, string(body2), "temporary credentials",
+		"Error should mention temporary credentials")
+}
+
+// TestSTSGetFederationTokenSuccess tests a successful GetFederationToken call
+// and verifies the returned credentials can be used to access S3
+func TestSTSGetFederationTokenSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	if !isGetFederationTokenImplemented(t) {
+		t.Skip("GetFederationToken not implemented")
+	}
+
+	accessKey, secretKey := getTestCredentials()
+
+	t.Run("basic_success", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"AppClient"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
+
+		if resp.StatusCode != http.StatusOK {
+			var errResp STSErrorTestResponse
+			_ = xml.Unmarshal(body, &errResp)
+			t.Fatalf("GetFederationToken failed: code=%s message=%s", errResp.Error.Code, errResp.Error.Message)
+		}
+
+		var stsResp GetFederationTokenTestResponse
+		require.NoError(t, xml.Unmarshal(body, &stsResp), "Parse response: %s", string(body))
+
+		creds := stsResp.Result.Credentials
+		assert.NotEmpty(t, creds.AccessKeyId)
+		assert.NotEmpty(t, creds.SecretAccessKey)
+		assert.NotEmpty(t, creds.SessionToken)
+		assert.NotEmpty(t, creds.Expiration)
+
+		fedUser := stsResp.Result.FederatedUser
+		assert.Contains(t, fedUser.Arn, "federated-user/AppClient")
+		assert.Contains(t, fedUser.FederatedUserId, "AppClient")
+	})
+
+	t.Run("with_custom_duration", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"GetFederationToken"},
+			"Version":         {"2011-06-15"},
+			"Name":            {"DurationTest"},
+			"DurationSeconds": {"3600"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
+
+		if resp.StatusCode == http.StatusOK {
+			var stsResp GetFederationTokenTestResponse
+			require.NoError(t, xml.Unmarshal(body, &stsResp))
+			assert.NotEmpty(t, stsResp.Result.Credentials.AccessKeyId)
+
+			// Verify expiration is roughly 1 hour from now
+			expTime, err := time.Parse(time.RFC3339, stsResp.Result.Credentials.Expiration)
+			require.NoError(t, err)
+			diff := time.Until(expTime)
+			assert.InDelta(t, 3600, diff.Seconds(), 60,
+				"Expiration should be ~1 hour from now")
+		}
+	})
+
+	t.Run("with_36_hour_duration", func(t *testing.T) {
+		// GetFederationToken allows up to 36 hours (unlike AssumeRole's 12h max)
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"GetFederationToken"},
+			"Version":         {"2011-06-15"},
+			"Name":            {"LongDuration"},
+			"DurationSeconds": {"129600"}, // 36 hours
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		if resp.StatusCode == http.StatusOK {
+			var stsResp GetFederationTokenTestResponse
+			require.NoError(t, xml.Unmarshal(body, &stsResp))
+
+			expTime, err := time.Parse(time.RFC3339, stsResp.Result.Credentials.Expiration)
+			require.NoError(t, err)
+			diff := time.Until(expTime)
+			assert.InDelta(t, 129600, diff.Seconds(), 60,
+				"Expiration should be ~36 hours from now")
+		} else {
+			// Duration should not cause a rejection
+			var errResp STSErrorTestResponse
+			_ = xml.Unmarshal(body, &errResp)
+			assert.NotContains(t, errResp.Error.Message, "DurationSeconds",
+				"36-hour duration should be accepted by GetFederationToken")
+		}
+	})
+}
+
+// TestSTSGetFederationTokenWithSessionPolicy tests that vended credentials
+// are scoped down by an inline session policy
+func TestSTSGetFederationTokenWithSessionPolicy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	if !isGetFederationTokenImplemented(t) {
+		t.Skip("GetFederationToken not implemented")
+	}
+
+	accessKey, secretKey := getTestCredentials()
+
+	// Create a test bucket using admin credentials
+	adminSess, err := session.NewSession(&aws.Config{
+		Region:           aws.String("us-east-1"),
+		Endpoint:         aws.String(TestSTSEndpoint),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
+	})
+	require.NoError(t, err)
+
+	adminS3 := s3.New(adminSess)
+	bucket := fmt.Sprintf("fed-token-test-%d", time.Now().UnixNano())
+
+	_, err = adminS3.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
+	require.NoError(t, err)
+	defer adminS3.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)})
+
+	_, err = adminS3.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String("test.txt"),
+		Body:   strings.NewReader("hello"),
+	})
+	require.NoError(t, err)
+	defer adminS3.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String("test.txt")})
+
+	// Get federated credentials with a session policy that only allows GetObject
+	sessionPolicy := fmt.Sprintf(`{
+		"Version": "2012-10-17",
+		"Statement": [{
+			"Effect": "Allow",
+			"Action": ["s3:GetObject"],
+			"Resource": ["arn:aws:s3:::%s/*"]
+		}]
+	}`, bucket)
+
+	resp, err := callSTSAPIWithSigV4(t, url.Values{
+		"Action":  {"GetFederationToken"},
+		"Version": {"2011-06-15"},
+		"Name":    {"ScopedClient"},
+		"Policy":  {sessionPolicy},
+	}, accessKey, secretKey)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+	t.Logf("GetFederationToken response: status=%d body=%s", resp.StatusCode, string(body))
+
+	if resp.StatusCode != http.StatusOK {
+		t.Skipf("GetFederationToken failed (may need IAM policy config): %s", string(body))
+	}
+
+	var stsResp GetFederationTokenTestResponse
+	require.NoError(t, xml.Unmarshal(body, &stsResp))
+
+	fedCreds := stsResp.Result.Credentials
+	require.NotEmpty(t, fedCreds.AccessKeyId)
+	require.NotEmpty(t, fedCreds.SessionToken)
+
+	// Create S3 client with the federated credentials
+	fedSess, err := session.NewSession(&aws.Config{
+		Region:           aws.String("us-east-1"),
+		Endpoint:         aws.String(TestSTSEndpoint),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+		Credentials: credentials.NewStaticCredentials(
+			fedCreds.AccessKeyId, fedCreds.SecretAccessKey, fedCreds.SessionToken),
+	})
+	require.NoError(t, err)
+
+	fedS3 := s3.New(fedSess)
+
+	// GetObject should succeed (allowed by session policy)
+	getResp, err := fedS3.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String("test.txt"),
+	})
+	if err == nil {
+		defer getResp.Body.Close()
+		t.Log("GetObject with federated credentials succeeded (as expected)")
+	} else {
+		t.Logf("GetObject with federated credentials: %v (session policy enforcement may vary)", err)
+	}
+
+	// PutObject should be denied (not allowed by session policy)
+	_, err = fedS3.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String("denied.txt"),
+		Body:   strings.NewReader("should fail"),
+	})
+	if err != nil {
+		t.Log("PutObject correctly denied with federated credentials")
+		assert.Contains(t, err.Error(), "AccessDenied",
+			"PutObject should be denied by session policy")
+	} else {
+		// Clean up if unexpectedly succeeded
+		adminS3.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String("denied.txt")})
+		t.Log("PutObject unexpectedly succeeded — session policy enforcement may not be active")
+	}
+}

test/s3/spark/issue_8285_repro_test.go

@@ -70,7 +70,8 @@ print("WRITE_COUNT=" + str(count))
 		"issue-8285/output/_temporary/0/",
 		"issue-8285/output/_temporary/0/_temporary/",
 	}
-	lingering := waitForObjectsToDisappear(t, env, "test", temporaryCandidates, 35*time.Second)
+	// Empty folder cleanup has a 2m default delay + 30s processor interval
+	lingering := waitForObjectsToDisappear(t, env, "test", temporaryCandidates, 3*time.Minute)
 	if len(lingering) > 0 {
 		t.Fatalf("issue #8285 regression detected: lingering temporary directories: %v", lingering)
 	}

test/s3/spark/setup_test.go

@@ -4,8 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"math/rand"
-	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -96,9 +94,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}
 
-	env.masterPort = mustFreePort(t, "Master")
-	env.filerPort = mustFreePort(t, "Filer")
-	env.s3Port = mustFreePort(t, "S3")
+	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
+	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
+	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
 
 	bindIP := testutil.FindBindIP()
 	iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)
@@ -135,14 +133,14 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	}
 	registerMiniProcess(env.masterProcess)
 
-	if !waitForPort(env.masterPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
 	}
-	if !waitForPort(env.filerPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
 	}
-	if !waitForPort(env.s3Port, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
 	}
 }
 
@@ -208,46 +206,6 @@ func (env *TestEnvironment) Cleanup(t *testing.T) {
 	}
 }
 
-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-
-	for i := 0; i < 200; i++ {
-		port := 20000 + rand.Intn(30000)
-		listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port))
-		if err != nil {
-			continue
-		}
-		_ = listener.Close()
-
-		grpcPort := port + 10000
-		if grpcPort > 65535 {
-			continue
-		}
-		grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", grpcPort))
-		if err != nil {
-			continue
-		}
-		_ = grpcListener.Close()
-		return port
-	}
-
-	t.Fatalf("failed to get free port for %s", name)
-	return 0
-}
-
-func waitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
-		if err == nil {
-			_ = conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}
-
 func runSparkPyScript(t *testing.T, container testcontainers.Container, script string, s3Port int) (int, string) {
 	t.Helper()
 

test/s3/sse/Makefile

@@ -98,8 +98,8 @@ start-seaweedfs: check-binary
 	# Create S3 configuration with SSE-KMS support
 	@printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}],"kms":{"type":"%s","configs":{"keyId":"%s","encryptionContext":{},"bucketKey":false}}}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" "$(KMS_TYPE)" "$(KMS_KEY_ID)" > /tmp/seaweedfs-sse-s3.json
 	
-	# Start weed mini
-	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(SEAWEEDFS_BINARY) mini \
+	# Start weed mini (WEED_S3_SSE_KEY enables SSE-S3 encryption)
+	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key $(SEAWEEDFS_BINARY) mini \
 		-dir=/tmp/seaweedfs-test-sse \
 		-s3.port=$(S3_PORT) \
 		-s3.config=/tmp/seaweedfs-sse-s3.json \
@@ -337,8 +337,9 @@ start-seaweedfs-ci: check-binary
 	     s3-config-template.json > /tmp/seaweedfs-s3.json
 	
 	# Start weed mini with embedded S3 using the JSON config (with verbose logging)
+	# WEED_S3_SSE_KEY enables SSE-S3 encryption for testing (KEK derived via HKDF)
 	@echo "Starting weed mini with embedded S3..."
-	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
+	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
 		-dir=/tmp/seaweedfs-test-sse \
 		-s3.port=$(S3_PORT) \
 		-s3.config=/tmp/seaweedfs-s3.json \
@@ -482,7 +483,7 @@ test-volume-encryption: build-weed
 	     -e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \
 	     s3-config-template.json > /tmp/seaweedfs-s3.json
 	@echo "Starting weed mini with S3 volume encryption..."
-	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
+	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
 		-dir=/tmp/seaweedfs-test-sse \
 		-s3.port=$(S3_PORT) \
 		-s3.config=/tmp/seaweedfs-s3.json \

test/s3/versioning/s3_versioning_pagination_stress_test.go

@@ -289,6 +289,157 @@ func TestVersioningPaginationMultipleObjectsManyVersions(t *testing.T) {
 	})
 }
 
+// TestVersioningPaginationDeepDirectoryHierarchy tests that paginated ListObjectVersions
+// correctly skips directory subtrees before the key-marker. This reproduces the
+// real-world scenario where Veeam backup objects are spread across many subdirectories
+// (e.g., Mailboxes/<uuid>/ItemsData/<file>) and pagination becomes exponentially
+// slower as the marker advances through the tree.
+//
+// Run with: ENABLE_STRESS_TESTS=true go test -v -run TestVersioningPaginationDeepDirectoryHierarchy -timeout 10m
+func TestVersioningPaginationDeepDirectoryHierarchy(t *testing.T) {
+	if os.Getenv("ENABLE_STRESS_TESTS") != "true" {
+		t.Skip("Skipping stress test. Set ENABLE_STRESS_TESTS=true to run.")
+	}
+
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	enableVersioning(t, client, bucketName)
+	checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
+
+	// Create a deep directory structure mimicking Veeam 365 backup layout:
+	// Backup/Organizations/<org>/Mailboxes/<mailbox>/ItemsData/<file>
+	numMailboxes := 20
+	filesPerMailbox := 5
+	totalObjects := numMailboxes * filesPerMailbox
+	orgPrefix := "Backup/Organizations/org-001/Mailboxes"
+
+	t.Logf("Creating %d objects across %d subdirectories (depth=6)...", totalObjects, numMailboxes)
+	startTime := time.Now()
+
+	allKeys := make([]string, 0, totalObjects)
+	for i := 0; i < numMailboxes; i++ {
+		mailboxId := fmt.Sprintf("mbx-%03d", i)
+		for j := 0; j < filesPerMailbox; j++ {
+			key := fmt.Sprintf("%s/%s/ItemsData/file-%03d.dat", orgPrefix, mailboxId, j)
+			_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(key),
+				Body:   strings.NewReader(fmt.Sprintf("content-%d-%d", i, j)),
+			})
+			require.NoError(t, err)
+			allKeys = append(allKeys, key)
+		}
+	}
+	t.Logf("Created %d objects in %v", totalObjects, time.Since(startTime))
+
+	// Test 1: Paginate through all versions with a broad prefix and small maxKeys.
+	// This forces multiple pages that must skip earlier subdirectory trees.
+	t.Run("PaginateAcrossSubdirectories", func(t *testing.T) {
+		maxKeys := int32(10) // Force many pages to exercise marker skipping
+		var allVersions []types.ObjectVersion
+		var keyMarker, versionIdMarker *string
+		pageCount := 0
+		pageStartTimes := make([]time.Duration, 0)
+
+		for {
+			pageStart := time.Now()
+			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+				Bucket:          aws.String(bucketName),
+				Prefix:          aws.String(orgPrefix + "/"),
+				MaxKeys:         aws.Int32(maxKeys),
+				KeyMarker:       keyMarker,
+				VersionIdMarker: versionIdMarker,
+			})
+			pageDuration := time.Since(pageStart)
+			require.NoError(t, err)
+			pageCount++
+			pageStartTimes = append(pageStartTimes, pageDuration)
+
+			allVersions = append(allVersions, resp.Versions...)
+			t.Logf("Page %d: %d versions in %v (marker: %v)",
+				pageCount, len(resp.Versions), pageDuration,
+				keyMarker)
+
+			if resp.IsTruncated == nil || !*resp.IsTruncated {
+				break
+			}
+			keyMarker = resp.NextKeyMarker
+			versionIdMarker = resp.NextVersionIdMarker
+		}
+
+		assert.Greater(t, pageCount, 1, "Should require multiple pages")
+
+		// Verify listed keys exactly match the keys we created (same elements, same order)
+		listedKeys := make([]string, 0, len(allVersions))
+		for _, v := range allVersions {
+			listedKeys = append(listedKeys, *v.Key)
+		}
+		assert.Equal(t, allKeys, listedKeys,
+			"Listed version keys should exactly match created keys")
+
+		// Check that later pages don't take dramatically longer than earlier ones.
+		// Before the fix, later pages were exponentially slower because they
+		// re-traversed the entire tree. With the fix, all pages should be similar.
+		if len(pageStartTimes) >= 4 {
+			firstQuarter := pageStartTimes[0]
+			lastQuarter := pageStartTimes[len(pageStartTimes)-1]
+			t.Logf("First page: %v, Last page: %v, Ratio: %.1fx",
+				firstQuarter, lastQuarter,
+				float64(lastQuarter)/float64(firstQuarter))
+			// Allow generous 10x ratio to avoid flakiness; before the fix
+			// the ratio was 100x+ on large datasets
+			if lastQuarter > firstQuarter*10 && lastQuarter > 500*time.Millisecond {
+				t.Errorf("Last page took %.1fx longer than first page (%v vs %v) — possible pagination regression",
+					float64(lastQuarter)/float64(firstQuarter), lastQuarter, firstQuarter)
+			}
+		}
+	})
+
+	// Test 2: Paginate with delimiter to verify CommonPrefixes interaction
+	t.Run("PaginateWithDelimiterAcrossSubdirs", func(t *testing.T) {
+		maxKeys := int32(5)
+		var allPrefixes []string
+		var keyMarker *string
+		pageCount := 0
+
+		for {
+			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+				Bucket:    aws.String(bucketName),
+				Prefix:    aws.String(orgPrefix + "/"),
+				Delimiter: aws.String("/"),
+				MaxKeys:   aws.Int32(maxKeys),
+				KeyMarker: keyMarker,
+			})
+			require.NoError(t, err)
+			pageCount++
+
+			for _, cp := range resp.CommonPrefixes {
+				allPrefixes = append(allPrefixes, *cp.Prefix)
+			}
+
+			if resp.IsTruncated == nil || !*resp.IsTruncated {
+				break
+			}
+			keyMarker = resp.NextKeyMarker
+		}
+
+		assert.Greater(t, pageCount, 1, "Should require multiple pages with maxKeys=%d", maxKeys)
+
+		// Build the exact expected prefixes
+		expectedPrefixes := make([]string, 0, numMailboxes)
+		for i := 0; i < numMailboxes; i++ {
+			expectedPrefixes = append(expectedPrefixes,
+				fmt.Sprintf("%s/mbx-%03d/", orgPrefix, i))
+		}
+		assert.Equal(t, expectedPrefixes, allPrefixes,
+			"CommonPrefixes should exactly match expected mailbox prefixes")
+	})
+}
+
 // listAllVersions is a helper to list all versions of a specific object using pagination
 func listAllVersions(t *testing.T, client *s3.Client, bucketName, objectKey string) []types.ObjectVersion {
 	var allVersions []types.ObjectVersion

test/s3tables/catalog_risingwave/setup_test.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -115,11 +114,11 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}
 
-	env.masterPort = mustFreePort(t, "Master")
-	env.filerPort = mustFreePort(t, "Filer")
-	env.s3Port = mustFreePort(t, "S3")
-	env.icebergRestPort = mustFreePort(t, "Iceberg")
-	env.risingwavePort = mustFreePort(t, "RisingWave")
+	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
+	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
+	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
+	env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")
+	env.risingwavePort = testutil.MustFreeMiniPort(t, "RisingWave")
 
 	env.bindIP = testutil.FindBindIP()
 
@@ -161,62 +160,20 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	registerMiniProcess(env.masterProcess)
 
 	// Wait for all services to be ready
-	if !waitForPort(env.masterPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
 	}
-	if !waitForPort(env.filerPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
 	}
-	if !waitForPort(env.s3Port, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
 	}
-	if !waitForPort(env.icebergRestPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - iceberg rest port %d not listening", env.icebergRestPort)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/v1/config", env.icebergRestPort), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - iceberg rest endpoint http://127.0.0.1:%d/v1/config not responding", env.icebergRestPort)
 	}
 }
 
-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-	minPort := 10000
-	maxPort := 55000 // Ensure port+10000 < 65535
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-
-	for i := 0; i < 1000; i++ {
-		port := minPort + r.Intn(maxPort-minPort)
-
-		// Check http port
-		ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
-		if err != nil {
-			continue
-		}
-		ln.Close()
-
-		// Check grpc port (weed mini uses port+10000)
-		ln2, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port+10000))
-		if err != nil {
-			continue
-		}
-		ln2.Close()
-
-		return port
-	}
-	t.Fatalf("failed to find a free port < %d for %s after 1000 attempts", maxPort, name)
-	return 0
-}
-
-func waitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
-		if err == nil {
-			conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}
-
 func (env *TestEnvironment) StartRisingWave(t *testing.T) {
 	t.Helper()
 
@@ -253,7 +210,7 @@ func (env *TestEnvironment) StartRisingWave(t *testing.T) {
 	}
 
 	// Wait for RisingWave port to be open on host
-	if !waitForPort(env.risingwavePort, 120*time.Second) {
+	if !testutil.WaitForPort(env.risingwavePort, 120*time.Second) {
 		t.Fatalf("timed out waiting for RisingWave port %d to be open", env.risingwavePort)
 	}
 

test/s3tables/catalog_spark/setup_test.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
-	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -91,10 +90,10 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}
 
-	env.masterPort = mustFreePort(t, "Master")
-	env.filerPort = mustFreePort(t, "Filer")
-	env.s3Port = mustFreePort(t, "S3")
-	env.icebergRestPort = mustFreePort(t, "Iceberg")
+	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
+	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
+	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
+	env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")
 
 	bindIP := testutil.FindBindIP()
 
@@ -127,58 +126,20 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	registerMiniProcess(env.masterProcess)
 
 	// Wait for all services to be ready
-	if !waitForPort(env.masterPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
 	}
-	if !waitForPort(env.filerPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
 	}
-	if !waitForPort(env.s3Port, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
 	}
-	if !waitForPort(env.icebergRestPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - iceberg rest port %d not listening", env.icebergRestPort)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/v1/config", env.icebergRestPort), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - iceberg rest endpoint http://127.0.0.1:%d/v1/config not responding", env.icebergRestPort)
 	}
 }
 
-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-
-	for i := 0; i < 200; i++ {
-		port := 20000 + rand.Intn(30000)
-		listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port))
-		if err != nil {
-			continue
-		}
-		listener.Close()
-		grpcPort := port + 10000
-		if grpcPort > 65535 {
-			continue
-		}
-		grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", grpcPort))
-		if err != nil {
-			continue
-		}
-		grpcListener.Close()
-		return port
-	}
-	t.Fatalf("failed to get free port for %s", name)
-	return 0
-}
-
-func waitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
-		if err == nil {
-			conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}
-
 func (env *TestEnvironment) writeSparkConfig(t *testing.T, catalogBucket string) string {
 	t.Helper()
 

test/s3tables/catalog_spark/spark_operations_test.go

@@ -277,3 +277,80 @@ print(f"Count at snapshot: {count}")
 
 	t.Logf(">>> Time travel test passed")
 }
+
+// TestSparkMultiLevelNamespace tests that multi-level namespaces produce correct
+// S3 paths (dot-separated) so that Spark can read back the data it writes.
+// Regression test for https://github.com/seaweedfs/seaweedfs/issues/8959
+func TestSparkMultiLevelNamespace(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env, _, _ := setupSparkTestEnv(t)
+
+	// Use a two-level namespace like "analytics.daily"
+	nsLevel1 := "analytics_" + randomString(4)
+	nsLevel2 := "daily_" + randomString(4)
+	multiNs := fmt.Sprintf("%s.%s", nsLevel1, nsLevel2)
+	tableName := "events_" + randomString(4)
+
+	// Create multi-level namespace
+	t.Logf(">>> Creating multi-level namespace: %s", multiNs)
+	createNsSQL := fmt.Sprintf(`
+spark.sql("CREATE NAMESPACE iceberg.%s")
+print("Namespace created")
+`, multiNs)
+	output := runSparkPySQL(t, env.sparkContainer, createNsSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Namespace created") {
+		t.Fatalf("multi-level namespace creation failed, output: %s", output)
+	}
+
+	// Create table under multi-level namespace
+	t.Logf(">>> Creating table under multi-level namespace")
+	createTableSQL := fmt.Sprintf(`
+spark.sql("""
+CREATE TABLE iceberg.%s.%s (
+    id INT,
+    event STRING,
+    ts TIMESTAMP
+)
+USING iceberg
+""")
+print("Table created")
+`, multiNs, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, createTableSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Table created") {
+		t.Fatalf("table creation under multi-level namespace failed, output: %s", output)
+	}
+
+	// Insert data
+	t.Logf(">>> Inserting data into multi-level namespace table")
+	insertSQL := fmt.Sprintf(`
+spark.sql("""
+INSERT INTO iceberg.%s.%s VALUES
+    (1, 'click', TIMESTAMP '2025-01-01 00:00:00'),
+    (2, 'view',  TIMESTAMP '2025-01-01 01:00:00'),
+    (3, 'click', TIMESTAMP '2025-01-02 00:00:00')
+""")
+print("Data inserted")
+`, multiNs, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, insertSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Data inserted") {
+		t.Fatalf("data insertion failed, output: %s", output)
+	}
+
+	// Query data back — this is the key test: if the namespace path separator
+	// was wrong (\x1F instead of "."), Spark would not find the data files.
+	t.Logf(">>> Querying data from multi-level namespace table")
+	querySQL := fmt.Sprintf(`
+result = spark.sql("SELECT COUNT(*) as count FROM iceberg.%s.%s")
+count = result.collect()[0]['count']
+print(f"Row count: {count}")
+`, multiNs, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, querySQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Row count: 3") {
+		t.Errorf("expected row count 3 from multi-level namespace table, got output: %s", output)
+	}
+
+	t.Logf(">>> Multi-level namespace test passed")
+}

test/s3tables/catalog_trino/trino_catalog_test.go

@@ -82,6 +82,101 @@ func TestTrinoIcebergCatalog(t *testing.T) {
 	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SHOW TABLES FROM iceberg.%s", schemaName))
 }
 
+// TestTrinoMultiLevelNamespace tests that multi-level namespaces (dot-separated)
+// produce correct S3 paths so Trino can read back data it writes.
+// Regression test for https://github.com/seaweedfs/seaweedfs/issues/8959
+func TestTrinoMultiLevelNamespace(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	if !env.dockerAvailable {
+		t.Skip("Docker not available, skipping Trino integration test")
+	}
+
+	t.Logf(">>> Starting SeaweedFS...")
+	env.StartSeaweedFS(t)
+
+	tableBucket := "iceberg-tables"
+	createTableBucket(t, env, tableBucket)
+
+	configDir := env.writeTrinoConfig(t, tableBucket, withNestedNamespace())
+	env.startTrinoContainer(t, configDir)
+	waitForTrino(t, env.trinoContainer, 60*time.Second)
+
+	// Use a two-level namespace: "analytics.daily"
+	nsLevel1 := "analytics_" + randomString(4)
+	nsLevel2 := "daily_" + randomString(4)
+	flatNs := fmt.Sprintf("%s.%s", nsLevel1, nsLevel2)
+	// Trino uses double-quoted schema names for multi-level namespaces
+	multiNs := fmt.Sprintf(`"%s"`, flatNs)
+	tableName := "events_" + randomString(4)
+
+	// Create multi-level namespace (schema)
+	t.Logf(">>> Creating multi-level schema: %s", flatNs)
+	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", multiNs))
+
+	// Verify the schema shows up
+	output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
+	if !strings.Contains(output, flatNs) {
+		t.Fatalf("Expected schema %s in output:\n%s", flatNs, output)
+	}
+
+	// Create table with explicit location to avoid non-empty location conflict.
+	// The location uses the dot-separated namespace — if #8959 regresses
+	// (unit separator instead of dot), data would be written to the wrong path.
+	tableLocation := fmt.Sprintf("s3://%s/%s/%s_%s", tableBucket, flatNs, tableName, randomString(6))
+	t.Logf(">>> Creating table at location: %s", tableLocation)
+	createSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS iceberg.%s.%s (
+		id INTEGER,
+		event VARCHAR,
+		ts TIMESTAMP(6)
+	) WITH (
+		format = 'PARQUET',
+		location = '%s'
+	)`, multiNs, tableName, tableLocation)
+	runTrinoSQLAllowExists(t, env.trinoContainer, createSQL)
+
+	// Insert data
+	t.Logf(">>> Inserting data into multi-level namespace table")
+	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`
+		INSERT INTO iceberg.%s.%s VALUES
+			(1, 'click', TIMESTAMP '2025-01-01 00:00:00'),
+			(2, 'view',  TIMESTAMP '2025-01-01 01:00:00'),
+			(3, 'click', TIMESTAMP '2025-01-02 00:00:00')
+	`, multiNs, tableName))
+
+	// Query data back — if the namespace path separator were wrong (\x1F
+	// instead of "."), the metadata location would point to a non-existent
+	// S3 path and this query would fail.
+	t.Logf(">>> Querying data from multi-level namespace table")
+	countOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
+		"SELECT count(*) FROM iceberg.%s.%s", multiNs, tableName))
+	rowCount := mustParseCSVInt64(t, countOutput)
+	if rowCount != 3 {
+		t.Fatalf("expected row count 3, got %d", rowCount)
+	}
+
+	// Verify the S3 file path contains the dot-separated namespace, not \x1F.
+	filesOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
+		`SELECT file_path FROM iceberg.%s."%s$files" LIMIT 1`, multiNs, tableName))
+	filePath := strings.TrimSpace(filesOutput)
+	if filePath == "" {
+		t.Fatalf("expected at least one data file, got empty output")
+	}
+	if !strings.Contains(filePath, flatNs+"/") {
+		t.Errorf("expected file path to contain dot-separated namespace %q, got: %s", flatNs, filePath)
+	}
+	if strings.Contains(filePath, "\x1F") {
+		t.Errorf("file path contains unit separator (\\x1F), expected dot separator: %s", filePath)
+	}
+
+	t.Logf(">>> Trino multi-level namespace test passed")
+}
+
 func NewTestEnvironment(t *testing.T) *TestEnvironment {
 	t.Helper()
 
@@ -332,18 +427,32 @@ func testIcebergRestAPI(t *testing.T, env *TestEnvironment) {
 	}
 }
 
-func (env *TestEnvironment) writeTrinoConfig(t *testing.T, warehouseBucket string) string {
+func (env *TestEnvironment) writeTrinoConfig(t *testing.T, warehouseBucket string, opts ...func(*trinoConfigOptions)) string {
 	t.Helper()
 
-	configDir := filepath.Join(env.dataDir, "trino")
+	o := trinoConfigOptions{}
+	for _, fn := range opts {
+		fn(&o)
+	}
+
+	dirName := "trino"
+	if o.nestedNamespace {
+		dirName = "trino-nested"
+	}
+	configDir := filepath.Join(env.dataDir, dirName)
 	if err := os.MkdirAll(configDir, 0755); err != nil {
 		t.Fatalf("Failed to create Trino config dir: %v", err)
 	}
 
+	nestedLine := ""
+	if o.nestedNamespace {
+		nestedLine = "\niceberg.rest-catalog.nested-namespace-enabled=true"
+	}
+
 	config := fmt.Sprintf(`connector.name=iceberg
 iceberg.catalog.type=rest
 iceberg.rest-catalog.uri=http://host.docker.internal:%d
-iceberg.rest-catalog.warehouse=s3://%s
+iceberg.rest-catalog.warehouse=s3://%s%s
 iceberg.file-format=PARQUET
 iceberg.unique-table-location=true
 
@@ -358,7 +467,7 @@ s3.region=us-west-2
 
 # REST catalog authentication
 iceberg.rest-catalog.security=SIGV4
-`, env.icebergPort, warehouseBucket, env.s3Port, env.accessKey, env.secretKey)
+`, env.icebergPort, warehouseBucket, nestedLine, env.s3Port, env.accessKey, env.secretKey)
 
 	if err := os.WriteFile(filepath.Join(configDir, "iceberg.properties"), []byte(config), 0644); err != nil {
 		t.Fatalf("Failed to write Trino config: %v", err)
@@ -367,6 +476,14 @@ iceberg.rest-catalog.security=SIGV4
 	return configDir
 }
 
+type trinoConfigOptions struct {
+	nestedNamespace bool
+}
+
+func withNestedNamespace() func(*trinoConfigOptions) {
+	return func(o *trinoConfigOptions) { o.nestedNamespace = true }
+}
+
 func (env *TestEnvironment) startTrinoContainer(t *testing.T, configDir string) {
 	t.Helper()
 

test/s3tables/lakekeeper/lakekeeper_test.go

@@ -115,10 +115,13 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 
 	bindIP := testutil.FindBindIP()
 
-	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
-	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
-	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
-	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
+	// Allocate all ports in a single batch to prevent the OS from recycling
+	// a released port, which can cause two services to get the same port.
+	ports := testutil.MustAllocatePorts(t, 8)
+	masterPort, masterGrpcPort := ports[0], ports[1]
+	volumePort, volumeGrpcPort := ports[2], ports[3]
+	filerPort, filerGrpcPort := ports[4], ports[5]
+	s3Port, s3GrpcPort := ports[6], ports[7]
 
 	return &TestEnvironment{
 		seaweedDir:     seaweedDir,

test/s3tables/polaris/polaris_env_test.go

@@ -89,11 +89,14 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 
 	bindIP := testutil.FindBindIP()
 
-	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
-	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
-	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
-	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
-	polarisPort, polarisAdminPort := testutil.MustFreePortPair(t, "Polaris")
+	// Allocate all ports in a single batch to prevent the OS from recycling
+	// a released port, which can cause two services to get the same port.
+	ports := testutil.MustAllocatePorts(t, 10)
+	masterPort, masterGrpcPort := ports[0], ports[1]
+	volumePort, volumeGrpcPort := ports[2], ports[3]
+	filerPort, filerGrpcPort := ports[4], ports[5]
+	s3Port, s3GrpcPort := ports[6], ports[7]
+	polarisPort, polarisAdminPort := ports[8], ports[9]
 
 	return &TestEnvironment{
 		seaweedDir:       seaweedDir,

test/s3tables/sts_integration/sts_integration_test.go

@@ -93,10 +93,13 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 
 	bindIP := testutil.FindBindIP()
 
-	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
-	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
-	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
-	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3") // Changed to use testutil.MustFreePortPair
+	// Allocate all ports in a single batch to prevent the OS from recycling
+	// a released port, which can cause two services to get the same port.
+	ports := testutil.MustAllocatePorts(t, 8)
+	masterPort, masterGrpcPort := ports[0], ports[1]
+	volumePort, volumeGrpcPort := ports[2], ports[3]
+	filerPort, filerGrpcPort := ports[4], ports[5]
+	s3Port, s3GrpcPort := ports[6], ports[7]
 
 	return &TestEnvironment{
 		seaweedDir:      seaweedDir,

test/s3tables/testutil/docker.go

@@ -2,6 +2,7 @@ package testutil
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"net/http"
 	"os/exec"
@@ -14,33 +15,44 @@ func HasDocker() bool {
 	return cmd.Run() == nil
 }
 
+// MustFreePortPair is a convenience wrapper for tests that only need a single pair.
+// Prefer MustAllocatePorts when allocating multiple pairs to guarantee uniqueness.
 func MustFreePortPair(t *testing.T, name string) (int, int) {
-	httpPort, grpcPort, err := findAvailablePortPair()
-	if err != nil {
-		t.Fatalf("Failed to get free port pair for %s: %v", name, err)
-	}
-	return httpPort, grpcPort
+	ports := MustAllocatePorts(t, 2)
+	return ports[0], ports[1]
 }
 
-func findAvailablePortPair() (int, int, error) {
-	httpPort, err := GetFreePort()
+// MustAllocatePorts allocates count unique free ports atomically.
+// All listeners are held open until every port is obtained, preventing
+// the OS from recycling a port between successive allocations.
+func MustAllocatePorts(t *testing.T, count int) []int {
+	t.Helper()
+	ports, err := AllocatePorts(count)
 	if err != nil {
-		return 0, 0, err
+		t.Fatalf("Failed to allocate %d free ports: %v", count, err)
 	}
-	grpcPort, err := GetFreePort()
-	if err != nil {
-		return 0, 0, err
-	}
-	return httpPort, grpcPort, nil
+	return ports
 }
 
-func GetFreePort() (int, error) {
-	listener, err := net.Listen("tcp", "0.0.0.0:0")
-	if err != nil {
-		return 0, err
+// AllocatePorts allocates count unique free ports atomically.
+func AllocatePorts(count int) ([]int, error) {
+	listeners := make([]net.Listener, 0, count)
+	ports := make([]int, 0, count)
+	for i := 0; i < count; i++ {
+		l, err := net.Listen("tcp", "0.0.0.0:0")
+		if err != nil {
+			for _, ll := range listeners {
+				_ = ll.Close()
+			}
+			return nil, err
+		}
+		listeners = append(listeners, l)
+		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
 	}
-	defer listener.Close()
-	return listener.Addr().(*net.TCPAddr).Port, nil
+	for _, l := range listeners {
+		_ = l.Close()
+	}
+	return ports, nil
 }
 
 func WaitForService(url string, timeout time.Duration) bool {
@@ -64,3 +76,17 @@ func WaitForService(url string, timeout time.Duration) bool {
 		}
 	}
 }
+
+func WaitForPort(port int, timeout time.Duration) bool {
+	deadline := time.Now().Add(timeout)
+	address := fmt.Sprintf("127.0.0.1:%d", port)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", address, 500*time.Millisecond)
+		if err == nil {
+			_ = conn.Close()
+			return true
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return false
+}

test/s3tables/testutil/weed_mini.go

@@ -2,11 +2,16 @@ package testutil
 
 import (
 	"fmt"
+	"math/rand"
 	"net"
 	"os"
 	"path/filepath"
+	"testing"
+	"time"
 )
 
+const SeaweedMiniStartupTimeout = 45 * time.Second
+
 func FindBindIP() string {
 	addrs, err := net.InterfaceAddrs()
 	if err != nil {
@@ -54,3 +59,35 @@ func WriteIAMConfig(dir, accessKey, secretKey string) (string, error) {
 	}
 	return iamConfigPath, nil
 }
+
+func MustFreeMiniPort(t *testing.T, name string) int {
+	t.Helper()
+
+	const (
+		minPort = 10000
+		maxPort = 55000
+	)
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	for i := 0; i < 1000; i++ {
+		port := minPort + r.Intn(maxPort-minPort)
+
+		listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+		if err != nil {
+			continue
+		}
+		_ = listener.Close()
+
+		grpcPort := port + 10000
+		grpcListener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
+		if err != nil {
+			continue
+		}
+		_ = grpcListener.Close()
+
+		return port
+	}
+
+	t.Fatalf("failed to get free weed mini port for %s", name)
+	return 0
+}

test/volume_server/framework/cluster_mixed.go

@@ -0,0 +1,368 @@
+package framework
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+// MixedVolumeCluster wraps a Go master + a mix of Go and Rust volume servers
+// for integration testing. Go servers occupy indices [0, goCount) and Rust
+// servers occupy indices [goCount, goCount+rustCount).
+type MixedVolumeCluster struct {
+	testingTB testing.TB
+	profile   matrix.Profile
+
+	weedBinary       string // Go weed binary (master + Go volume servers)
+	rustVolumeBinary string // Rust volume binary
+
+	baseDir   string
+	configDir string
+	logsDir   string
+	keepLogs  bool
+
+	masterPort     int
+	masterGrpcPort int
+
+	volumePorts     []int
+	volumeGrpcPorts []int
+	volumePubPorts  []int
+	isRust          []bool // which servers are Rust
+
+	masterCmd  *exec.Cmd
+	volumeCmds []*exec.Cmd
+
+	cleanupOnce sync.Once
+}
+
+// StartMixedVolumeCluster starts a cluster with 1 Go master, goCount Go volume
+// servers, and rustCount Rust volume servers. Go servers come first in the index.
+func StartMixedVolumeCluster(t testing.TB, profile matrix.Profile, goCount, rustCount int) *MixedVolumeCluster {
+	t.Helper()
+
+	if goCount < 0 || rustCount < 0 {
+		t.Fatalf("goCount and rustCount must be non-negative, got go=%d rust=%d", goCount, rustCount)
+	}
+	total := goCount + rustCount
+	if total < 2 {
+		t.Fatalf("need at least 2 volume servers, got %d", total)
+	}
+
+	weedBinary, err := FindOrBuildWeedBinary()
+	if err != nil {
+		t.Fatalf("resolve weed binary: %v", err)
+	}
+
+	// Only build the Rust binary when Rust servers are requested.
+	var rustBinary string
+	if rustCount > 0 {
+		rustBinary, err = FindOrBuildRustBinary()
+		if err != nil {
+			t.Skipf("skipping mixed cluster test: rust binary unavailable: %v", err)
+		}
+	}
+
+	baseDir, keepLogs, err := newWorkDir()
+	if err != nil {
+		t.Fatalf("create temp test directory: %v", err)
+	}
+
+	configDir := filepath.Join(baseDir, "config")
+	logsDir := filepath.Join(baseDir, "logs")
+	masterDataDir := filepath.Join(baseDir, "master")
+
+	dirs := []string{configDir, logsDir, masterDataDir}
+	for i := 0; i < total; i++ {
+		dirs = append(dirs, filepath.Join(baseDir, fmt.Sprintf("volume%d", i)))
+	}
+	for _, dir := range dirs {
+		if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
+			t.Fatalf("create %s: %v", dir, mkErr)
+		}
+	}
+
+	if err = writeSecurityConfig(configDir, profile); err != nil {
+		t.Fatalf("write security config: %v", err)
+	}
+
+	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	if err != nil {
+		t.Fatalf("allocate master port pair: %v", err)
+	}
+
+	// 2 ports per server (admin, grpc); add 1 more when public port is split out.
+	portsPerServer := 2
+	if profile.SplitPublicPort {
+		portsPerServer = 3
+	}
+	ports, err := allocatePorts(total * portsPerServer)
+	if err != nil {
+		t.Fatalf("allocate volume ports: %v", err)
+	}
+
+	isRust := make([]bool, total)
+	for i := goCount; i < total; i++ {
+		isRust[i] = true
+	}
+
+	c := &MixedVolumeCluster{
+		testingTB:        t,
+		profile:          profile,
+		weedBinary:       weedBinary,
+		rustVolumeBinary: rustBinary,
+		baseDir:          baseDir,
+		configDir:        configDir,
+		logsDir:          logsDir,
+		keepLogs:         keepLogs,
+		masterPort:       masterPort,
+		masterGrpcPort:   masterGrpcPort,
+		volumePorts:      make([]int, total),
+		volumeGrpcPorts:  make([]int, total),
+		volumePubPorts:   make([]int, total),
+		isRust:           isRust,
+		volumeCmds:       make([]*exec.Cmd, total),
+	}
+
+	for i := 0; i < total; i++ {
+		baseIdx := i * portsPerServer
+		c.volumePorts[i] = ports[baseIdx]
+		c.volumeGrpcPorts[i] = ports[baseIdx+1]
+		if profile.SplitPublicPort {
+			c.volumePubPorts[i] = ports[baseIdx+2]
+		} else {
+			c.volumePubPorts[i] = c.volumePorts[i] // reuse admin port
+		}
+	}
+
+	// Start master
+	if err = c.startMaster(masterDataDir); err != nil {
+		c.Stop()
+		t.Fatalf("start master: %v", err)
+	}
+	helper := &Cluster{logsDir: logsDir}
+	if err = helper.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
+		masterLog := helper.tailLog("master.log")
+		c.Stop()
+		t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
+	}
+
+	// Start volume servers
+	for i := 0; i < total; i++ {
+		volumeDataDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
+		if isRust[i] {
+			err = c.startRustVolume(i, volumeDataDir)
+		} else {
+			err = c.startGoVolume(i, volumeDataDir)
+		}
+		if err != nil {
+			logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
+			c.Stop()
+			t.Fatalf("start volume server %d (rust=%v): %v\nlog tail:\n%s", i, isRust[i], err, logTail)
+		}
+
+		// Rust uses /healthz, Go uses /status
+		healthURL := c.VolumeAdminURL(i) + "/status"
+		if isRust[i] {
+			healthURL = c.VolumeAdminURL(i) + "/healthz"
+		}
+		if err = helper.waitForHTTP(healthURL); err != nil {
+			logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
+			c.Stop()
+			t.Fatalf("wait for volume server %d readiness: %v\nlog tail:\n%s", i, err, logTail)
+		}
+		if err = helper.waitForTCP(c.VolumeGRPCAddress(i)); err != nil {
+			logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
+			c.Stop()
+			t.Fatalf("wait for volume server %d grpc readiness: %v\nlog tail:\n%s", i, err, logTail)
+		}
+	}
+
+	t.Cleanup(func() {
+		c.Stop()
+	})
+
+	return c
+}
+
+func (c *MixedVolumeCluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		for i := len(c.volumeCmds) - 1; i >= 0; i-- {
+			stopProcess(c.volumeCmds[i])
+		}
+		stopProcess(c.masterCmd)
+		if !c.keepLogs && !c.testingTB.Failed() {
+			_ = os.RemoveAll(c.baseDir)
+		} else if c.baseDir != "" {
+			c.testingTB.Logf("mixed volume server integration logs kept at %s", c.baseDir)
+		}
+	})
+}
+
+func (c *MixedVolumeCluster) startMaster(dataDir string) error {
+	logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"master",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.masterPort),
+		"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
+		"-mdir=" + dataDir,
+		"-peers=none",
+		"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
+		"-defaultReplication=000",
+	}
+
+	c.masterCmd = exec.Command(c.weedBinary, args...)
+	c.masterCmd.Dir = c.baseDir
+	c.masterCmd.Stdout = logFile
+	c.masterCmd.Stderr = logFile
+	if err = c.masterCmd.Start(); err != nil {
+		logFile.Close()
+		return err
+	}
+	logFile.Close() // child inherited the fd
+	return nil
+}
+
+func (c *MixedVolumeCluster) startGoVolume(index int, dataDir string) error {
+	logName := fmt.Sprintf("volume%d.log", index)
+	logFile, err := os.Create(filepath.Join(c.logsDir, logName))
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"volume",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.volumePorts[index]),
+		"-port.grpc=" + strconv.Itoa(c.volumeGrpcPorts[index]),
+		"-port.public=" + strconv.Itoa(c.volumePubPorts[index]),
+		"-dir=" + dataDir,
+		"-max=16",
+		"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
+		"-readMode=" + c.profile.ReadMode,
+		"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
+		"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
+	}
+	if c.profile.InflightUploadTimeout > 0 {
+		args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
+	}
+	if c.profile.InflightDownloadTimeout > 0 {
+		args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
+	}
+
+	cmd := exec.Command(c.weedBinary, args...)
+	cmd.Dir = c.baseDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err = cmd.Start(); err != nil {
+		logFile.Close()
+		return err
+	}
+	logFile.Close() // child inherited the fd
+	c.volumeCmds[index] = cmd
+	return nil
+}
+
+func (c *MixedVolumeCluster) startRustVolume(index int, dataDir string) error {
+	logName := fmt.Sprintf("volume%d.log", index)
+	logFile, err := os.Create(filepath.Join(c.logsDir, logName))
+	if err != nil {
+		return err
+	}
+
+	args := rustVolumeArgs(
+		c.profile,
+		c.configDir,
+		c.masterPort,
+		c.volumePorts[index],
+		c.volumeGrpcPorts[index],
+		c.volumePubPorts[index],
+		dataDir,
+	)
+
+	cmd := exec.Command(c.rustVolumeBinary, args...)
+	cmd.Dir = c.baseDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err = cmd.Start(); err != nil {
+		logFile.Close()
+		return err
+	}
+	logFile.Close() // child inherited the fd
+	c.volumeCmds[index] = cmd
+	return nil
+}
+
+// --- accessor methods (mirror MultiVolumeCluster) ---
+
+func (c *MixedVolumeCluster) MasterAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
+}
+
+func (c *MixedVolumeCluster) MasterURL() string {
+	return "http://" + c.MasterAddress()
+}
+
+func (c *MixedVolumeCluster) VolumeAdminAddress(index int) string {
+	if index < 0 || index >= len(c.volumePorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePorts[index]))
+}
+
+func (c *MixedVolumeCluster) VolumePublicAddress(index int) string {
+	if index < 0 || index >= len(c.volumePubPorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPorts[index]))
+}
+
+func (c *MixedVolumeCluster) VolumeGRPCAddress(index int) string {
+	if index < 0 || index >= len(c.volumeGrpcPorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPorts[index]))
+}
+
+func (c *MixedVolumeCluster) VolumeAdminURL(index int) string {
+	return "http://" + c.VolumeAdminAddress(index)
+}
+
+func (c *MixedVolumeCluster) VolumePublicURL(index int) string {
+	return "http://" + c.VolumePublicAddress(index)
+}
+
+func (c *MixedVolumeCluster) BaseDir() string {
+	return c.baseDir
+}
+
+// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort
+func (c *MixedVolumeCluster) VolumeServerAddress(index int) string {
+	if index < 0 || index >= len(c.volumePorts) {
+		return ""
+	}
+	return fmt.Sprintf("%s.%d", c.VolumeAdminAddress(index), c.volumeGrpcPorts[index])
+}
+
+func (c *MixedVolumeCluster) IsRust(index int) bool {
+	if index < 0 || index >= len(c.isRust) {
+		return false
+	}
+	return c.isRust[index]
+}

test/volume_server/framework/volume_fixture.go

@@ -80,6 +80,42 @@ func EnableMaintenanceMode(t testing.TB, ctx context.Context, client volume_serv
 	}
 }
 
+// CorruptDatFile truncates a volume's .dat file to just the superblock (8 bytes)
+// so that needle reads fail during a full scrub due to data file size mismatch.
+func CorruptDatFile(t testing.TB, baseDir string, volumeID uint32) {
+	t.Helper()
+	datPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.dat", volumeID))
+	// Truncate to superblock size only, removing all needle data.
+	if err := os.Truncate(datPath, 8); err != nil {
+		t.Fatalf("truncate dat file for corruption: %v", err)
+	}
+}
+
+// CorruptEcxFile appends garbage bytes to a volume's .ecx file on disk so
+// that CheckIndexFile detects a size mismatch during EC index scrub.
+func CorruptEcxFile(t testing.TB, baseDir string, volumeID uint32) {
+	t.Helper()
+	ecxPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.ecx", volumeID))
+	f, err := os.OpenFile(ecxPath, os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		t.Fatalf("open ecx file for corruption: %v", err)
+	}
+	defer f.Close()
+	if _, err := f.Write([]byte{0xDE, 0xAD}); err != nil {
+		t.Fatalf("corrupt ecx file: %v", err)
+	}
+}
+
+// CorruptEcShardFile truncates an EC shard file to 1 byte so that local shard
+// reads fail during an EC scrub.
+func CorruptEcShardFile(t testing.TB, baseDir string, volumeID uint32, shardID int) {
+	t.Helper()
+	shardPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.ec%02d", volumeID, shardID))
+	if err := os.Truncate(shardPath, 1); err != nil {
+		t.Fatalf("truncate EC shard file %s: %v", shardPath, err)
+	}
+}
+
 func ReadBytes(t testing.TB, client *http.Client, volumeURL, fid string) *http.Response {
 	t.Helper()
 

test/volume_server/grpc/ec_decode_ecj_test.go

@@ -0,0 +1,294 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// TestEcDecodePreservesDeletedNeedles verifies that needles deleted via
+// VolumeEcBlobDelete (recorded in .ecj) are correctly excluded from the
+// decoded volume produced by VolumeEcShardsToVolume.
+func TestEcDecodePreservesDeletedNeedles(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const (
+		volumeID = uint32(140)
+		keyA     = uint64(990020)
+		cookieA  = uint32(0xDA001122)
+		keyB     = uint64(990021)
+		cookieB  = uint32(0xDA003344)
+	)
+
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fidA := framework.NewFileID(volumeID, keyA, cookieA)
+	fidB := framework.NewFileID(volumeID, keyB, cookieB)
+	payloadA := []byte("needle-A-should-be-deleted-after-decode")
+	payloadB := []byte("needle-B-should-survive-decode")
+
+	// Upload two needles.
+	for _, tc := range []struct {
+		fid     string
+		payload []byte
+	}{
+		{fidA, payloadA},
+		{fidB, payloadB},
+	} {
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, tc.payload)
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s: expected 201, got %d", tc.fid, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// EC encode.
+	_, err := client.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate: %v", err)
+	}
+
+	// Mount all data shards so the EC volume is usable.
+	_, err = client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID, Collection: "",
+		ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount: %v", err)
+	}
+
+	// Delete needle A via EC path (writes to .ecj).
+	_, err = client.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId: volumeID, Collection: "",
+		FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete needle A: %v", err)
+	}
+
+	// Unmount the normal volume so decode writes fresh files.
+	_, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeUnmount: %v", err)
+	}
+
+	// Decode EC shards back to a normal volume.
+	_, err = client.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsToVolume: %v", err)
+	}
+
+	// Re-mount the decoded volume.
+	_, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeMount: %v", err)
+	}
+
+	// Needle A should be gone (deleted via .ecj before decode).
+	respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA)
+	bodyA := framework.ReadAllAndClose(t, respA)
+	if respA.StatusCode >= 500 {
+		t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
+	}
+	if respA.StatusCode != http.StatusNotFound {
+		t.Fatalf("needle A should be 404 after decode, got %d", respA.StatusCode)
+	}
+
+	// Needle B should still be readable.
+	respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB)
+	bodyB := framework.ReadAllAndClose(t, respB)
+	if respB.StatusCode != http.StatusOK {
+		t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
+	}
+	if string(bodyB) != string(payloadB) {
+		t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
+	}
+}
+
+// TestEcDecodeCollectsEcjFromPeer verifies that .ecj deletion entries from a
+// peer server that contributes no new data shards are still collected during
+// decode. This is the regression test for the fix in collectEcShards that
+// always copies .ecj from every shard location.
+//
+// Scenario:
+//   - Server 0 holds all 10 data shards (decode target).
+//   - Server 1 holds a copy of shard 0 (no new shards for server 0).
+//   - A needle is deleted ONLY on server 1 (server 0's .ecj is empty).
+//   - During decode on server 0, server 1's .ecj must be collected and applied.
+func TestEcDecodeCollectsEcjFromPeer(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMultiVolumeClusterAuto(t, matrix.P1(), 2)
+	conn0, client0 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, client1 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	const (
+		volumeID = uint32(141)
+		keyA     = uint64(990030)
+		cookieA  = uint32(0xDB001122)
+		keyB     = uint64(990031)
+		cookieB  = uint32(0xDB003344)
+	)
+
+	// Allocate and upload on server 0.
+	framework.AllocateVolume(t, client0, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fidA := framework.NewFileID(volumeID, keyA, cookieA)
+	fidB := framework.NewFileID(volumeID, keyB, cookieB)
+	payloadB := []byte("needle-B-should-survive-peer-ecj-decode")
+
+	resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA, []byte("needle-A-deleted-on-peer"))
+	_ = framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload A: expected 201, got %d", resp.StatusCode)
+	}
+	resp = framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB, payloadB)
+	_ = framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload B: expected 201, got %d", resp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// EC encode on server 0.
+	_, err := client0.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate on server 0: %v", err)
+	}
+
+	// Build the SourceDataNode address for server 0 (format: host:adminPort.grpcPort).
+	sourceDataNode := cluster.VolumeAdminAddress(0) + "." +
+		strings.Split(cluster.VolumeGRPCAddress(0), ":")[1]
+
+	// Copy shard 0 + ecx + ecj from server 0 → server 1.
+	_, err = client1.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: sourceDataNode,
+		ShardIds:       []uint32{0},
+		CopyEcxFile:    true,
+		CopyEcjFile:    true,
+		CopyVifFile:    true,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsCopy 0→1: %v", err)
+	}
+
+	// Mount shard 0 on server 1 so the EC volume can accept deletions.
+	_, err = client1.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID, Collection: "",
+		ShardIds: []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount on server 1: %v", err)
+	}
+
+	// Delete needle A on server 1 only (creates .ecj entry on server 1).
+	_, err = client1.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId: volumeID, Collection: "",
+		FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete needle A on server 1: %v", err)
+	}
+
+	// Mount all data shards on server 0 (the decode target).
+	_, err = client0.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID, Collection: "",
+		ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount on server 0: %v", err)
+	}
+
+	// Collect .ecj from server 1 → server 0 with NO new shard IDs.
+	// This is the critical path: server 1 has shard 0 which server 0 already
+	// has, so needToCopyShardsInfo would be empty. Before the fix in
+	// collectEcShards, this copy would be skipped entirely, losing server 1's
+	// deletion entries.
+	server1DataNode := cluster.VolumeAdminAddress(1) + "." +
+		strings.Split(cluster.VolumeGRPCAddress(1), ":")[1]
+
+	_, err = client0.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: server1DataNode,
+		ShardIds:       []uint32{}, // No new shards — just .ecj.
+		CopyEcxFile:    false,
+		CopyEcjFile:    true,
+		CopyVifFile:    false,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsCopy .ecj from server 1→0: %v", err)
+	}
+
+	// Unmount the normal volume before decode.
+	_, err = client0.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeUnmount on server 0: %v", err)
+	}
+
+	// Decode on server 0. RebuildEcxFile should see needle A's deletion from
+	// the .ecj that was collected from server 1.
+	_, err = client0.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsToVolume on server 0: %v", err)
+	}
+
+	// Re-mount the decoded normal volume.
+	_, err = client0.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeMount on server 0: %v", err)
+	}
+
+	// Needle A should be gone — its deletion was in server 1's .ecj.
+	respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA)
+	bodyA := framework.ReadAllAndClose(t, respA)
+	if respA.StatusCode >= 500 {
+		t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
+	}
+	if respA.StatusCode != http.StatusNotFound {
+		t.Fatalf("needle A should be 404 (ecj from peer), got %d", respA.StatusCode)
+	}
+
+	// Needle B should still be readable.
+	respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB)
+	bodyB := framework.ReadAllAndClose(t, respB)
+	if respB.StatusCode != http.StatusOK {
+		t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
+	}
+	if string(bodyB) != string(payloadB) {
+		t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
+	}
+}

test/volume_server/grpc/erasure_coding_test.go

@@ -752,6 +752,87 @@ func TestEcShardsCopyFromPeerSuccess(t *testing.T) {
 	}
 }
 
+// TestEcIndexConsistencyAfterEncode verifies that every needle indexed in .ecx
+// can be read back correctly from EC shards after VolumeEcShardsGenerate.
+// This catches the race condition fixed in this PR where .ecx could reference
+// data not present in EC shards.
+func TestEcIndexConsistencyAfterEncode(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(130)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+
+	// Upload multiple needles of varying sizes
+	type testNeedle struct {
+		fid     string
+		payload []byte
+	}
+	needles := []testNeedle{
+		{framework.NewFileID(volumeID, 1001, 0xAABB0001), []byte("small-needle-1")},
+		{framework.NewFileID(volumeID, 1002, 0xAABB0002), make([]byte, 1024)},     // 1KB
+		{framework.NewFileID(volumeID, 1003, 0xAABB0003), make([]byte, 64*1024)},   // 64KB
+		{framework.NewFileID(volumeID, 1004, 0xAABB0004), make([]byte, 256*1024)},  // 256KB
+		{framework.NewFileID(volumeID, 1005, 0xAABB0005), []byte("small-needle-2")},
+	}
+
+	// Fill larger payloads with recognizable data
+	for i := range needles {
+		for j := range needles[i].payload {
+			needles[i].payload[j] = byte(i*37 + j%251)
+		}
+	}
+
+	for _, n := range needles {
+		resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload)
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s expected 201, got %d", n.fid, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// EC encode
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	// Mount all data shards so reads go through the EC path
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount failed: %v", err)
+	}
+
+	// Read every needle back from EC shards and verify payload
+	for _, n := range needles {
+		readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid)
+		readBody := framework.ReadAllAndClose(t, readResp)
+		if readResp.StatusCode != http.StatusOK {
+			t.Fatalf("EC read %s expected 200, got %d", n.fid, readResp.StatusCode)
+		}
+		if string(readBody) != string(n.payload) {
+			t.Fatalf("EC read %s payload mismatch: got %d bytes, want %d bytes", n.fid, len(readBody), len(n.payload))
+		}
+	}
+}
+
 func TestEcShardsCopyFailsWhenSourceUnavailable(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test in short mode")

test/volume_server/grpc/mixed_balance_test.go

@@ -0,0 +1,567 @@
+package volume_server_grpc_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+// deleteAndWaitForTombstone issues an HTTP DELETE for the given fid on
+// volumeURL, asserts a successful response, then polls with GET until
+// the file returns 404 (tombstone visible) or the timeout elapses.
+func deleteAndWaitForTombstone(t *testing.T, httpClient *http.Client, volumeURL, fid string) {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/%s", volumeURL, fid), nil)
+	if err != nil {
+		t.Fatalf("build delete request for %s: %v", fid, err)
+	}
+	resp := framework.DoRequest(t, httpClient, req)
+	framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK {
+		t.Fatalf("delete %s: expected 200 or 202, got %d", fid, resp.StatusCode)
+	}
+	// Poll until GET returns 404 (tombstone flushed to disk).
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		getResp := framework.ReadBytes(t, httpClient, volumeURL, fid)
+		status := getResp.StatusCode
+		framework.ReadAllAndClose(t, getResp)
+		if status == http.StatusNotFound {
+			return
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Fatalf("delete %s: tombstone not visible after 5s", fid)
+}
+
+// TestMixedBalanceCopyGoToRust verifies that VolumeCopy works from a Go volume
+// server to a Rust volume server. This is the core operation behind volume
+// balancing in a mixed Go+Rust cluster.
+func TestMixedBalanceCopyGoToRust(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
+
+	// server 0 = Go, server 1 = Rust
+	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	httpClient := framework.NewHTTPClient()
+
+	const volumeID = uint32(50)
+
+	// Allocate volume on Go server and upload test data
+	framework.AllocateVolume(t, goClient, volumeID, "")
+
+	testFiles := []struct {
+		key    uint64
+		cookie uint32
+		data   []byte
+	}{
+		{1, 0xAABBCCDD, []byte("hello from Go server")},
+		{2, 0x11223344, []byte("second file for balance test")},
+		{3, 0xDEADBEEF, make([]byte, 4096)}, // larger file
+	}
+
+	for _, f := range testFiles {
+		fid := framework.NewFileID(volumeID, f.key, f.cookie)
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, f.data)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// Read source volume status before copy
+	sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
+	}
+	t.Logf("Source: dat=%d idx=%d files=%d version=%d",
+		sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
+		sourceStatus.GetFileCount(), sourceStatus.GetVersion())
+
+	// Copy volume from Go (server 0) to Rust (server 1)
+	copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy call failed: %v", err)
+	}
+
+	var lastAppendAtNs uint64
+	for {
+		resp, recvErr := copyStream.Recv()
+		if recvErr != nil {
+			if recvErr != io.EOF {
+				t.Fatalf("VolumeCopy recv error: %v", recvErr)
+			}
+			break
+		}
+		if resp.GetLastAppendAtNs() != 0 {
+			lastAppendAtNs = resp.GetLastAppendAtNs()
+		}
+	}
+	t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
+
+	// Verify: read volume status from Rust server
+	targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
+	}
+	t.Logf("Target: dat=%d idx=%d files=%d version=%d",
+		targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
+		targetStatus.GetFileCount(), targetStatus.GetVersion())
+
+	if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
+		t.Fatalf("dat file size mismatch: source=%d target=%d",
+			sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
+	}
+	if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
+		t.Fatalf("idx file size mismatch: source=%d target=%d",
+			sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
+	}
+	if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
+		t.Fatalf("file count mismatch: source=%d target=%d",
+			sourceStatus.GetFileCount(), targetStatus.GetFileCount())
+	}
+
+	// Verify data can be read from Rust server
+	for _, f := range testFiles {
+		fid := framework.NewFileID(volumeID, f.key, f.cookie)
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("read %s from Rust server: expected 200, got %d", fid, resp.StatusCode)
+		}
+		if !bytes.Equal(body, f.data) {
+			t.Fatalf("read %s from Rust server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
+		}
+	}
+}
+
+// TestMixedBalanceCopyRustToGo verifies that VolumeCopy works from a Rust
+// volume server to a Go volume server.
+func TestMixedBalanceCopyRustToGo(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
+
+	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	httpClient := framework.NewHTTPClient()
+
+	const volumeID = uint32(51)
+
+	// Allocate volume on Rust server and upload test data
+	framework.AllocateVolume(t, rustClient, volumeID, "")
+
+	testFiles := []struct {
+		key    uint64
+		cookie uint32
+		data   []byte
+	}{
+		{1, 0xAABBCCDD, []byte("hello from Rust server")},
+		{2, 0x11223344, []byte("second file for reverse balance")},
+	}
+
+	for _, f := range testFiles {
+		fid := framework.NewFileID(volumeID, f.key, f.cookie)
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid, f.data)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	sourceStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
+	}
+
+	// Copy volume from Rust (server 1) to Go (server 0)
+	copyStream, err := goClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: cluster.VolumeServerAddress(1),
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy call failed: %v", err)
+	}
+	for {
+		_, recvErr := copyStream.Recv()
+		if recvErr != nil {
+			if recvErr != io.EOF {
+				t.Fatalf("VolumeCopy recv error: %v", recvErr)
+			}
+			break
+		}
+	}
+
+	targetStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
+	}
+
+	if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
+		t.Fatalf("dat file size mismatch: source=%d target=%d",
+			sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
+	}
+	if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
+		t.Fatalf("idx file size mismatch: source=%d target=%d",
+			sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
+	}
+	if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
+		t.Fatalf("file count mismatch: source=%d target=%d",
+			sourceStatus.GetFileCount(), targetStatus.GetFileCount())
+	}
+
+	// Verify data can be read from Go server
+	for _, f := range testFiles {
+		fid := framework.NewFileID(volumeID, f.key, f.cookie)
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("read %s from Go server: expected 200, got %d", fid, resp.StatusCode)
+		}
+		if !bytes.Equal(body, f.data) {
+			t.Fatalf("read %s from Go server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
+		}
+	}
+}
+
+// TestMixedBalanceCopyWithDeletes verifies that VolumeCopy correctly handles
+// volumes that have both active and deleted needles.
+func TestMixedBalanceCopyWithDeletes(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
+
+	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	httpClient := framework.NewHTTPClient()
+
+	const volumeID = uint32(52)
+	framework.AllocateVolume(t, goClient, volumeID, "")
+
+	// Upload files
+	for i := uint64(1); i <= 5; i++ {
+		fid := framework.NewFileID(volumeID, i, 0x12340000+uint32(i))
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("file-%d", i)))
+		framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
+		}
+	}
+
+	// Delete some files and wait for tombstones to be visible
+	for _, key := range []uint64{2, 4} {
+		fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
+		deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0), fid)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus: %v", err)
+	}
+	t.Logf("Source after deletes: dat=%d idx=%d files=%d",
+		sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
+		sourceStatus.GetFileCount())
+
+	// Copy volume from Go to Rust
+	copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy: %v", err)
+	}
+	var lastAppendAtNs uint64
+	for {
+		resp, recvErr := copyStream.Recv()
+		if recvErr != nil {
+			if recvErr != io.EOF {
+				t.Fatalf("VolumeCopy recv error: %v", recvErr)
+			}
+			break
+		}
+		if resp.GetLastAppendAtNs() != 0 {
+			lastAppendAtNs = resp.GetLastAppendAtNs()
+		}
+	}
+	if lastAppendAtNs == 0 {
+		t.Fatalf("VolumeCopy did not return a lastAppendAtNs timestamp")
+	}
+	t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
+
+	targetStatusAfterCopy, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Rust: %v", err)
+	}
+	t.Logf("Target after copy: dat=%d idx=%d files=%d",
+		targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterCopy.GetIdxFileSize(),
+		targetStatusAfterCopy.GetFileCount())
+
+	if sourceStatus.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
+		t.Fatalf("dat file size mismatch: source=%d target=%d",
+			sourceStatus.GetDatFileSize(), targetStatusAfterCopy.GetDatFileSize())
+	}
+	if sourceStatus.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
+		t.Fatalf("idx file size mismatch: source=%d target=%d",
+			sourceStatus.GetIdxFileSize(), targetStatusAfterCopy.GetIdxFileSize())
+	}
+
+	// Tail from the copy checkpoint — source is unchanged (deletes happened
+	// before copy), so tailing should not append any data.
+	_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
+		VolumeId:           volumeID,
+		SinceNs:            lastAppendAtNs,
+		IdleTimeoutSeconds: 3,
+		SourceVolumeServer: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeTailReceiver: %v", err)
+	}
+
+	targetStatusAfterTail, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus after tail: %v", err)
+	}
+	if targetStatusAfterTail.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
+		t.Fatalf("dat grew after tail: before=%d after=%d",
+			targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterTail.GetDatFileSize())
+	}
+	if targetStatusAfterTail.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
+		t.Fatalf("idx grew after tail: before=%d after=%d",
+			targetStatusAfterCopy.GetIdxFileSize(), targetStatusAfterTail.GetIdxFileSize())
+	}
+
+	// Verify surviving files are readable from Rust
+	for _, key := range []uint64{1, 3, 5} {
+		fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("read surviving file %d from Rust: expected 200, got %d", key, resp.StatusCode)
+		}
+		expected := fmt.Sprintf("file-%d", key)
+		if string(body) != expected {
+			t.Fatalf("data mismatch for file %d: got %q want %q", key, body, expected)
+		}
+	}
+
+	// Verify deleted files return 404
+	for _, key := range []uint64{2, 4} {
+		fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusNotFound {
+			t.Fatalf("read deleted file %d from Rust: expected 404, got %d", key, resp.StatusCode)
+		}
+	}
+}
+
+// TestMixedBalanceFullMoveGoToRust exercises the complete volume balance move
+// flow: mark readonly → copy → tail → verify sizes → delete source.
+// This mirrors the steps in balance_task.go Execute().
+func TestMixedBalanceFullMoveGoToRust(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
+
+	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	httpClient := framework.NewHTTPClient()
+
+	const volumeID = uint32(53)
+	framework.AllocateVolume(t, goClient, volumeID, "")
+
+	// Upload test data
+	for i := uint64(1); i <= 5; i++ {
+		fid := framework.NewFileID(volumeID, i, 0xABCD0000+uint32(i))
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("balance-move-file-%d", i)))
+		framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// Step 1: Copy volume to Rust server (source stays writable so we can
+	// delete after copy to exercise the tail tombstone path)
+	copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy: %v", err)
+	}
+	var lastAppendAtNs uint64
+	for {
+		resp, recvErr := copyStream.Recv()
+		if recvErr != nil {
+			if recvErr != io.EOF {
+				t.Fatalf("VolumeCopy recv: %v", recvErr)
+			}
+			break
+		}
+		if resp.GetLastAppendAtNs() != 0 {
+			lastAppendAtNs = resp.GetLastAppendAtNs()
+		}
+	}
+	t.Logf("Copy done, lastAppendAtNs=%d", lastAppendAtNs)
+
+	// Step 2: Delete file 3 on the source AFTER copy. This creates a
+	// tombstone needle that the tail step must propagate to the Rust server.
+	deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0),
+		framework.NewFileID(volumeID, 3, 0xABCD0003))
+
+	// Step 3: Mark source readonly so no further writes arrive during tail
+	_, err = goClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("mark readonly: %v", err)
+	}
+
+	// Read source status (the reference for post-tail verification)
+	sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("read source status: %v", err)
+	}
+	t.Logf("Source: dat=%d idx=%d files=%d",
+		sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
+		sourceStatus.GetFileCount())
+
+	// Step 4: Tail for updates — this must pick up the delete tombstone
+	_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
+		VolumeId:           volumeID,
+		SinceNs:            lastAppendAtNs,
+		IdleTimeoutSeconds: 5,
+		SourceVolumeServer: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeTailReceiver: %v", err)
+	}
+
+	// Step 5: Verify dat/idx sizes match after tail.
+	// We compare file sizes (byte-level correctness) rather than file_count
+	// because the tail writes the tombstone via the write path (which
+	// increments file_count) while the source used the delete path (which
+	// only increments deletion_count). This is consistent with Go behavior.
+	targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("read target status: %v", err)
+	}
+	t.Logf("Target: dat=%d idx=%d files=%d",
+		targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
+		targetStatus.GetFileCount())
+
+	if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
+		t.Fatalf("dat size mismatch after tail: source=%d target=%d",
+			sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
+	}
+	if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
+		t.Fatalf("idx size mismatch after tail: source=%d target=%d",
+			sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
+	}
+
+	// Step 6: Delete volume from source
+	_, err = goClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("delete source volume: %v", err)
+	}
+
+	// Verify source volume is gone
+	_, err = goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err == nil {
+		t.Fatalf("expected error reading deleted source volume, got nil")
+	}
+	t.Logf("Source volume deleted successfully (error as expected: %v)", err)
+
+	// Verify all surviving data is readable from Rust (the new home)
+	for _, key := range []uint64{1, 2, 4, 5} {
+		fid := framework.NewFileID(volumeID, key, 0xABCD0000+uint32(key))
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("read file %d from Rust after move: expected 200, got %d", key, resp.StatusCode)
+		}
+		expected := fmt.Sprintf("balance-move-file-%d", key)
+		if string(body) != expected {
+			t.Fatalf("data mismatch for file %d after move: got %q want %q", key, body, expected)
+		}
+	}
+
+	// Verify deleted file is 404 on Rust (tombstone propagated via tail)
+	{
+		fid := framework.NewFileID(volumeID, 3, 0xABCD0003)
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusNotFound {
+			t.Fatalf("deleted file 3 on Rust: expected 404, got %d", resp.StatusCode)
+		}
+	}
+
+	t.Logf("Full balance move completed: volume %d moved from Go to Rust, source purged", volumeID)
+}

test/volume_server/grpc/scrub_integration_test.go

@@ -0,0 +1,416 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// --- Normal volume scrub tests ---
+
+func TestScrubVolumeFullHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(200)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("data-one")))
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 2, 2), []byte("data-two")))
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 3, 3), []byte("data-three")))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume FULL on healthy volume failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 1 {
+		t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
+	}
+	if resp.GetTotalFiles() != 3 {
+		t.Fatalf("expected total_files=3, got %d", resp.GetTotalFiles())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
+	}
+}
+
+func TestScrubVolumeFullCorruptData(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(201)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("important data")))
+
+	framework.CorruptDatFile(t, clusterHarness.BaseDir(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume FULL on corrupt volume failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after data corruption, got none")
+	}
+	if len(resp.GetDetails()) == 0 {
+		t.Fatalf("expected error details for corrupt volume")
+	}
+}
+
+func TestScrubVolumeMixedHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const healthyVol = uint32(202)
+	const corruptVol = uint32(203)
+	framework.AllocateVolume(t, grpcClient, healthyVol, "")
+	framework.AllocateVolume(t, grpcClient, corruptVol, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(healthyVol, 1, 1), []byte("healthy")))
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(corruptVol, 1, 1), []byte("will corrupt")))
+
+	framework.CorruptIndexFile(t, clusterHarness.BaseDir(), corruptVol)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{healthyVol, corruptVol},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume INDEX on mixed volumes failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 2 {
+		t.Fatalf("expected total_volumes=2, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 1 {
+		t.Fatalf("expected exactly 1 broken volume, got %v", resp.GetBrokenVolumeIds())
+	}
+	if resp.GetBrokenVolumeIds()[0] != corruptVol {
+		t.Fatalf("expected broken volume %d, got %d", corruptVol, resp.GetBrokenVolumeIds()[0])
+	}
+}
+
+func TestScrubVolumeMissingVolumeReturnsError(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{99999},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err == nil {
+		t.Fatalf("ScrubVolume should fail for missing volume")
+	}
+}
+
+// --- EC volume scrub tests ---
+
+// ecSetup creates a volume, uploads data, generates EC shards, and mounts all of them.
+func ecSetup(t *testing.T, grpcClient volume_server_pb.VolumeServerClient, httpClient *http.Client, volumeURL string, volumeID uint32) {
+	t.Helper()
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 1, 0xABCD0001)
+	uploadResp := framework.UploadBytes(t, httpClient, volumeURL, fid, []byte("ec-scrub-test-data-payload"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	allShards := make([]uint32, erasure_coding.TotalShardsCount)
+	for i := range allShards {
+		allShards[i] = uint32(i)
+	}
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   allShards,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount all shards failed: %v", err)
+	}
+}
+
+func TestScrubEcVolumeIndexHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(210)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume INDEX on healthy volume failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 1 {
+		t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
+	}
+}
+
+func TestScrubEcVolumeLocalHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(211)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_LOCAL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume LOCAL on healthy volume failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 1 {
+		t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
+	}
+	if resp.GetTotalFiles() != 1 {
+		t.Fatalf("expected total_files=1, got %d", resp.GetTotalFiles())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
+	}
+	if len(resp.GetBrokenShardInfos()) != 0 {
+		t.Fatalf("expected no broken shards, got %v", resp.GetBrokenShardInfos())
+	}
+}
+
+func TestScrubEcVolumeLocalCorruptShard(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(212)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	// Corrupt shard 0 by truncating it.
+	framework.CorruptEcShardFile(t, clusterHarness.BaseDir(), volumeID, 0)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_LOCAL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume LOCAL on corrupt shard failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after shard corruption")
+	}
+	if len(resp.GetBrokenShardInfos()) == 0 {
+		t.Fatalf("expected broken shard info after shard corruption")
+	}
+	// Verify all reported broken shards belong to the corrupted volume.
+	for _, si := range resp.GetBrokenShardInfos() {
+		if si.GetVolumeId() != volumeID {
+			t.Fatalf("broken shard info for unexpected volume %d, want %d", si.GetVolumeId(), volumeID)
+		}
+	}
+}
+
+func TestScrubEcVolumeAutoSelectWithEcPresent(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeIDA = uint32(213)
+	const volumeIDB = uint32(214)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeIDA)
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeIDB)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// Auto-select (empty VolumeIds) should find both EC volumes.
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		Mode: volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume auto-select failed: %v", err)
+	}
+	if resp.GetTotalVolumes() < 2 {
+		t.Fatalf("expected at least 2 EC volumes via auto-select, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v", resp.GetBrokenVolumeIds())
+	}
+}
+
+func TestScrubEcVolumeUnsupportedMode(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(215)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode(99),
+	})
+	if err == nil {
+		t.Fatalf("ScrubEcVolume should fail for unsupported mode")
+	}
+}
+
+func TestScrubEcVolumeIndexCorruptEcx(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(216)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 1, 0xABCD0001)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-ecx-corrupt-test"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// Generate EC shards (this creates the .ecx file on disk).
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	// Corrupt the .ecx file BEFORE mounting, so the corrupted size is loaded.
+	framework.CorruptEcxFile(t, clusterHarness.BaseDir(), volumeID)
+
+	// Now mount shards - the ecx file size will reflect the corruption.
+	allShards := make([]uint32, erasure_coding.TotalShardsCount)
+	for i := range allShards {
+		allShards[i] = uint32(i)
+	}
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   allShards,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount failed: %v", err)
+	}
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume INDEX on corrupt ecx failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after ECX corruption")
+	}
+}

weed/admin/dash/admin_data.go

@@ -45,6 +45,7 @@ type ObjectStoreUser struct {
 	SecretKey   string   `json:"secret_key"`
 	Permissions []string `json:"permissions"`
 	PolicyNames []string `json:"policy_names"`
+	IsStatic    bool     `json:"is_static"` // loaded from static config file, not editable
 }
 
 type ObjectStoreUsersData struct {

weed/admin/dash/admin_server.go

@@ -867,6 +867,24 @@ func (s *AdminServer) DeleteS3Bucket(bucketName string) error {
 	})
 }
 
+// IsStaticUser checks if a user is a static identity by loading the
+// configuration from the credential manager and checking the IsStatic flag.
+func (s *AdminServer) IsStaticUser(username string) bool {
+	if s.credentialManager == nil {
+		return false
+	}
+	s3cfg, err := s.credentialManager.LoadConfiguration(context.Background())
+	if err != nil {
+		return false
+	}
+	for _, ident := range s3cfg.Identities {
+		if ident.Name == username {
+			return ident.IsStatic
+		}
+	}
+	return false
+}
+
 // GetObjectStoreUsers retrieves object store users from identity.json
 func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUser, error) {
 	if s.credentialManager == nil {
@@ -890,6 +908,7 @@ func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUse
 		user := ObjectStoreUser{
 			Username:    identity.Name,
 			Permissions: identity.Actions,
+			IsStatic:    identity.IsStatic,
 		}
 
 		// Set email from account if available
@@ -943,7 +962,7 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 			leaderCount++
 		}
 
-		masterMap[master.Address] = masterInfo
+		masterMap[masterInfo.Address] = masterInfo
 	}
 
 	// Then, get additional master information from Raft cluster
@@ -955,11 +974,11 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 
 		// Process each raft server
 		for _, server := range resp.ClusterServers {
-			address := server.Address
-			httpAddress := pb.ServerAddress(address).ToHttpAddress()
+			// Raft stores gRPC addresses, convert to HTTP address
+			httpAddress := pb.GrpcAddressToServerAddress(server.Address)
 
 			// Update existing master info or create new one
-			if masterInfo, exists := masterMap[address]; exists {
+			if masterInfo, exists := masterMap[httpAddress]; exists {
 				// Update existing master with raft data
 				masterInfo.IsLeader = server.IsLeader
 				masterInfo.Suffrage = server.Suffrage
@@ -970,7 +989,7 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 					IsLeader: server.IsLeader,
 					Suffrage: server.Suffrage,
 				}
-				masterMap[address] = masterInfo
+				masterMap[httpAddress] = masterInfo
 			}
 
 			if server.IsLeader {
@@ -1646,159 +1665,6 @@ func (as *AdminServer) GetConfigPersistence() *ConfigPersistence {
 	return as.configPersistence
 }
 
-// convertJSONToMaintenanceConfig converts JSON map to protobuf MaintenanceConfig
-func convertJSONToMaintenanceConfig(jsonConfig map[string]interface{}) (*maintenance.MaintenanceConfig, error) {
-	config := &maintenance.MaintenanceConfig{}
-
-	// Helper function to get int32 from interface{}
-	getInt32 := func(key string) (int32, error) {
-		if val, ok := jsonConfig[key]; ok {
-			switch v := val.(type) {
-			case int:
-				return int32(v), nil
-			case int32:
-				return v, nil
-			case int64:
-				return int32(v), nil
-			case float64:
-				return int32(v), nil
-			default:
-				return 0, fmt.Errorf("invalid type for %s: expected number, got %T", key, v)
-			}
-		}
-		return 0, nil
-	}
-
-	// Helper function to get bool from interface{}
-	getBool := func(key string) bool {
-		if val, ok := jsonConfig[key]; ok {
-			if b, ok := val.(bool); ok {
-				return b
-			}
-		}
-		return false
-	}
-
-	var err error
-
-	// Convert basic fields
-	config.Enabled = getBool("enabled")
-
-	if config.ScanIntervalSeconds, err = getInt32("scan_interval_seconds"); err != nil {
-		return nil, err
-	}
-	if config.WorkerTimeoutSeconds, err = getInt32("worker_timeout_seconds"); err != nil {
-		return nil, err
-	}
-	if config.TaskTimeoutSeconds, err = getInt32("task_timeout_seconds"); err != nil {
-		return nil, err
-	}
-	if config.RetryDelaySeconds, err = getInt32("retry_delay_seconds"); err != nil {
-		return nil, err
-	}
-	if config.MaxRetries, err = getInt32("max_retries"); err != nil {
-		return nil, err
-	}
-	if config.CleanupIntervalSeconds, err = getInt32("cleanup_interval_seconds"); err != nil {
-		return nil, err
-	}
-	if config.TaskRetentionSeconds, err = getInt32("task_retention_seconds"); err != nil {
-		return nil, err
-	}
-
-	// Convert policy if present
-	if policyData, ok := jsonConfig["policy"]; ok {
-		if policyMap, ok := policyData.(map[string]interface{}); ok {
-			policy := &maintenance.MaintenancePolicy{}
-
-			if globalMaxConcurrent, err := getInt32FromMap(policyMap, "global_max_concurrent"); err != nil {
-				return nil, err
-			} else {
-				policy.GlobalMaxConcurrent = globalMaxConcurrent
-			}
-
-			if defaultRepeatIntervalSeconds, err := getInt32FromMap(policyMap, "default_repeat_interval_seconds"); err != nil {
-				return nil, err
-			} else {
-				policy.DefaultRepeatIntervalSeconds = defaultRepeatIntervalSeconds
-			}
-
-			if defaultCheckIntervalSeconds, err := getInt32FromMap(policyMap, "default_check_interval_seconds"); err != nil {
-				return nil, err
-			} else {
-				policy.DefaultCheckIntervalSeconds = defaultCheckIntervalSeconds
-			}
-
-			// Convert task policies if present
-			if taskPoliciesData, ok := policyMap["task_policies"]; ok {
-				if taskPoliciesMap, ok := taskPoliciesData.(map[string]interface{}); ok {
-					policy.TaskPolicies = make(map[string]*maintenance.TaskPolicy)
-
-					for taskType, taskPolicyData := range taskPoliciesMap {
-						if taskPolicyMap, ok := taskPolicyData.(map[string]interface{}); ok {
-							taskPolicy := &maintenance.TaskPolicy{}
-
-							taskPolicy.Enabled = getBoolFromMap(taskPolicyMap, "enabled")
-
-							if maxConcurrent, err := getInt32FromMap(taskPolicyMap, "max_concurrent"); err != nil {
-								return nil, err
-							} else {
-								taskPolicy.MaxConcurrent = maxConcurrent
-							}
-
-							if repeatIntervalSeconds, err := getInt32FromMap(taskPolicyMap, "repeat_interval_seconds"); err != nil {
-								return nil, err
-							} else {
-								taskPolicy.RepeatIntervalSeconds = repeatIntervalSeconds
-							}
-
-							if checkIntervalSeconds, err := getInt32FromMap(taskPolicyMap, "check_interval_seconds"); err != nil {
-								return nil, err
-							} else {
-								taskPolicy.CheckIntervalSeconds = checkIntervalSeconds
-							}
-
-							policy.TaskPolicies[taskType] = taskPolicy
-						}
-					}
-				}
-			}
-
-			config.Policy = policy
-		}
-	}
-
-	return config, nil
-}
-
-// Helper functions for map conversion
-func getInt32FromMap(m map[string]interface{}, key string) (int32, error) {
-	if val, ok := m[key]; ok {
-		switch v := val.(type) {
-		case int:
-			return int32(v), nil
-		case int32:
-			return v, nil
-		case int64:
-			return int32(v), nil
-		case float64:
-			return int32(v), nil
-		default:
-			return 0, fmt.Errorf("invalid type for %s: expected number, got %T", key, v)
-		}
-	}
-	return 0, nil
-}
-
-func getBoolFromMap(m map[string]interface{}, key string) bool {
-	if val, ok := m[key]; ok {
-		if b, ok := val.(bool); ok {
-			return b
-		}
-	}
-	return false
-}
-
 type collectionStats struct {
 	PhysicalSize int64
 	LogicalSize  int64

weed/admin/dash/bucket_management.go

@@ -361,26 +361,6 @@ func normalizeQuotaUnit(unit string) (string, error) {
 	}
 }
 
-// Helper function to convert bytes to appropriate unit and size
-func convertBytesToQuota(bytes int64) (int64, string) {
-	if bytes == 0 {
-		return 0, "MB"
-	}
-
-	// Convert to TB if >= 1TB
-	if bytes >= 1024*1024*1024*1024 && bytes%(1024*1024*1024*1024) == 0 {
-		return bytes / (1024 * 1024 * 1024 * 1024), "TB"
-	}
-
-	// Convert to GB if >= 1GB
-	if bytes >= 1024*1024*1024 && bytes%(1024*1024*1024) == 0 {
-		return bytes / (1024 * 1024 * 1024), "GB"
-	}
-
-	// Convert to MB (default)
-	return bytes / (1024 * 1024), "MB"
-}
-
 // SetBucketQuota sets the quota for a bucket
 func (s *AdminServer) SetBucketQuota(bucketName string, quotaBytes int64, quotaEnabled bool) error {
 	return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

weed/admin/dash/config_persistence.go

@@ -1091,7 +1091,7 @@ func (cp *ConfigPersistence) loadTaskStateLocked(taskID string) (*maintenance.Ma
 	// Convert protobuf to maintenance task
 	task := cp.protobufToMaintenanceTask(taskStateFile.Task)
 
-	glog.V(2).Infof("Loaded task state for task %s from %s", taskID, taskFilePath)
+	glog.V(3).Infof("Loaded task state for task %s from %s", taskID, taskFilePath)
 	return task, nil
 }
 
@@ -1135,6 +1135,43 @@ func (cp *ConfigPersistence) loadAllTaskStatesLocked() ([]*maintenance.Maintenan
 	return tasks, nil
 }
 
+// DeleteAllTaskStates removes all task state .pb files from disk without reading them.
+// Used at startup to clean up stale files from previous runs — the scanner will
+// re-detect any tasks that are still needed from live cluster state.
+func (cp *ConfigPersistence) DeleteAllTaskStates() error {
+	cp.tasksMu.Lock()
+	defer cp.tasksMu.Unlock()
+
+	if cp.dataDir == "" {
+		return nil
+	}
+
+	tasksDir := filepath.Join(cp.dataDir, TasksSubdir)
+	entries, err := os.ReadDir(tasksDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("failed to read tasks directory: %w", err)
+	}
+
+	var removed int
+	for _, entry := range entries {
+		if !entry.IsDir() && filepath.Ext(entry.Name()) == ".pb" {
+			if err := os.Remove(filepath.Join(tasksDir, entry.Name())); err != nil && !os.IsNotExist(err) {
+				glog.Warningf("Failed to delete task file %s: %v", entry.Name(), err)
+			} else {
+				removed++
+			}
+		}
+	}
+
+	if removed > 0 {
+		glog.Infof("Cleaned up %d stale task files from disk", removed)
+	}
+	return nil
+}
+
 // DeleteTaskState removes a task state file from disk
 func (cp *ConfigPersistence) DeleteTaskState(taskID string) error {
 	cp.tasksMu.Lock()

weed/admin/dash/ec_shard_management.go

@@ -506,18 +506,6 @@ func getShardCount(ecIndexBits uint32) int {
 	return count
 }
 
-// getMissingShards returns a slice of missing shard IDs for a volume
-// Assumes default 10+4 EC configuration (14 total shards)
-func getMissingShards(ecIndexBits uint32) []int {
-	var missing []int
-	for i := 0; i < erasure_coding.TotalShardsCount; i++ {
-		if (ecIndexBits & (1 << uint(i))) == 0 {
-			missing = append(missing, i)
-		}
-	}
-	return missing
-}
-
 // sortEcShards sorts EC shards based on the specified field and order
 func sortEcShards(shards []EcShardWithInfo, sortBy string, sortOrder string) {
 	sort.Slice(shards, func(i, j int) bool {

weed/admin/dash/mq_management.go

@@ -430,67 +430,6 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
 	return offsets, nil
 }
 
-// convertRecordTypeToSchemaFields converts a protobuf RecordType to SchemaFieldInfo slice
-func convertRecordTypeToSchemaFields(recordType *schema_pb.RecordType) []SchemaFieldInfo {
-	var schemaFields []SchemaFieldInfo
-
-	if recordType == nil || recordType.Fields == nil {
-		return schemaFields
-	}
-
-	for _, field := range recordType.Fields {
-		schemaField := SchemaFieldInfo{
-			Name:     field.Name,
-			Type:     getFieldTypeString(field.Type),
-			Required: field.IsRequired,
-		}
-		schemaFields = append(schemaFields, schemaField)
-	}
-
-	return schemaFields
-}
-
-// getFieldTypeString converts a protobuf Type to a human-readable string
-func getFieldTypeString(fieldType *schema_pb.Type) string {
-	if fieldType == nil {
-		return "unknown"
-	}
-
-	switch kind := fieldType.Kind.(type) {
-	case *schema_pb.Type_ScalarType:
-		return getScalarTypeString(kind.ScalarType)
-	case *schema_pb.Type_RecordType:
-		return "record"
-	case *schema_pb.Type_ListType:
-		elementType := getFieldTypeString(kind.ListType.ElementType)
-		return fmt.Sprintf("list<%s>", elementType)
-	default:
-		return "unknown"
-	}
-}
-
-// getScalarTypeString converts a protobuf ScalarType to a string
-func getScalarTypeString(scalarType schema_pb.ScalarType) string {
-	switch scalarType {
-	case schema_pb.ScalarType_BOOL:
-		return "bool"
-	case schema_pb.ScalarType_INT32:
-		return "int32"
-	case schema_pb.ScalarType_INT64:
-		return "int64"
-	case schema_pb.ScalarType_FLOAT:
-		return "float"
-	case schema_pb.ScalarType_DOUBLE:
-		return "double"
-	case schema_pb.ScalarType_BYTES:
-		return "bytes"
-	case schema_pb.ScalarType_STRING:
-		return "string"
-	default:
-		return "unknown"
-	}
-}
-
 // convertTopicPublishers converts protobuf TopicPublisher slice to PublisherInfo slice
 func convertTopicPublishers(publishers []*mq_pb.TopicPublisher) []PublisherInfo {
 	publisherInfos := make([]PublisherInfo, 0, len(publishers))

weed/admin/dash/plugin_api.go

@@ -2,8 +2,6 @@ package dash
 
 import (
 	"context"
-	"crypto/rand"
-	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -851,43 +849,6 @@ func normalizeTimeout(timeoutSeconds int, defaultTimeout, maxTimeout time.Durati
 	return timeout
 }
 
-func buildJobSpecFromProposal(jobType string, proposal *plugin_pb.JobProposal, index int) *plugin_pb.JobSpec {
-	now := timestamppb.Now()
-	suffix := make([]byte, 4)
-	if _, err := rand.Read(suffix); err != nil {
-		// Fallback to simpler ID if rand fails
-		suffix = []byte(fmt.Sprintf("%d", index))
-	}
-	jobID := fmt.Sprintf("%s-%d-%s", jobType, now.AsTime().UnixNano(), hex.EncodeToString(suffix))
-
-	jobSpec := &plugin_pb.JobSpec{
-		JobId:      jobID,
-		JobType:    jobType,
-		Priority:   plugin_pb.JobPriority_JOB_PRIORITY_NORMAL,
-		CreatedAt:  now,
-		Labels:     make(map[string]string),
-		Parameters: make(map[string]*plugin_pb.ConfigValue),
-		DedupeKey:  "",
-	}
-
-	if proposal != nil {
-		jobSpec.Summary = proposal.Summary
-		jobSpec.Detail = proposal.Detail
-		if proposal.Priority != plugin_pb.JobPriority_JOB_PRIORITY_UNSPECIFIED {
-			jobSpec.Priority = proposal.Priority
-		}
-		jobSpec.DedupeKey = proposal.DedupeKey
-		jobSpec.Parameters = plugin.CloneConfigValueMap(proposal.Parameters)
-		if proposal.Labels != nil {
-			for k, v := range proposal.Labels {
-				jobSpec.Labels[k] = v
-			}
-		}
-	}
-
-	return jobSpec
-}
-
 func applyDescriptorDefaultsToPersistedConfig(
 	config *plugin_pb.PersistedJobTypeConfig,
 	descriptor *plugin_pb.JobTypeDescriptor,

weed/admin/dash/plugin_api_test.go

@@ -115,32 +115,6 @@ func TestExpirePluginJobAPI(t *testing.T) {
 	})
 }
 
-func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
-	t.Parallel()
-
-	proposal := &plugin_pb.JobProposal{
-		ProposalId: "vacuum-2",
-		DedupeKey:  "vacuum:2",
-		JobType:    "vacuum",
-	}
-
-	jobA := buildJobSpecFromProposal("vacuum", proposal, 0)
-	jobB := buildJobSpecFromProposal("vacuum", proposal, 1)
-
-	if jobA.JobId == proposal.ProposalId {
-		t.Fatalf("job id must not reuse proposal id: %s", jobA.JobId)
-	}
-	if jobB.JobId == proposal.ProposalId {
-		t.Fatalf("job id must not reuse proposal id: %s", jobB.JobId)
-	}
-	if jobA.JobId == jobB.JobId {
-		t.Fatalf("job ids must be unique across jobs: %s", jobA.JobId)
-	}
-	if jobA.DedupeKey != proposal.DedupeKey {
-		t.Fatalf("dedupe key must be preserved: got=%s want=%s", jobA.DedupeKey, proposal.DedupeKey)
-	}
-}
-
 func TestApplyDescriptorDefaultsToPersistedConfigBackfillsAdminDefaults(t *testing.T) {
 	t.Parallel()
 

weed/admin/dash/types.go

@@ -447,11 +447,6 @@ type QueueStats = maintenance.QueueStats
 type WorkerDetailsData = maintenance.WorkerDetailsData
 type WorkerPerformance = maintenance.WorkerPerformance
 
-// GetTaskIcon returns the icon CSS class for a task type from its UI provider
-func GetTaskIcon(taskType MaintenanceTaskType) string {
-	return maintenance.GetTaskIcon(taskType)
-}
-
 // Status constants (these are still static)
 const (
 	TaskStatusPending    = maintenance.TaskStatusPending

weed/admin/dash/user_management.go

@@ -3,13 +3,13 @@ package dash
 import (
 	"context"
 	"crypto/rand"
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"strings"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/credential"
+	"github.com/seaweedfs/seaweedfs/weed/iam"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 )
 
@@ -175,7 +175,7 @@ func (s *AdminServer) GetObjectStoreUserDetails(username string) (*UserDetails,
 
 	ctx := context.Background()
 
-	// Get user using credential manager
+	// Get user using credential manager (resolves static users via filer gRPC)
 	identity, err := s.credentialManager.GetUser(ctx, username)
 	if err != nil {
 		if err == credential.ErrUserNotFound {
@@ -435,10 +435,13 @@ func generateAccessKey() string {
 }
 
 func generateSecretKey() string {
-	// Generate 40-character secret key (AWS standard)
-	b := make([]byte, 30) // 30 bytes = 40 characters in base64
-	rand.Read(b)
-	return base64.StdEncoding.EncodeToString(b)
+	// Use the IAM helper to generate URL-safe secret keys (no +, / characters)
+	// that won't break S3 signature authentication
+	key, err := iam.GenerateSecretAccessKey()
+	if err != nil {
+		panic(fmt.Sprintf("failed to generate secret key: %v", err))
+	}
+	return key
 }
 
 func generateAccountId() string {

weed/admin/dash/user_management_test.go

@@ -74,9 +74,16 @@ func TestGenerateSecretKey(t *testing.T) {
 	key1 := generateSecretKey()
 	key2 := generateSecretKey()
 
-	// Check length (base64 encoding of 30 bytes = 40 characters)
-	if len(key1) != 40 {
-		t.Errorf("Expected secret key length 40, got %d", len(key1))
+	// Check length (IAM standard secret key length)
+	if len(key1) != 42 {
+		t.Errorf("Expected secret key length 42, got %d", len(key1))
+	}
+
+	// Check that key contains only URL-safe characters (no +, /)
+	for _, c := range key1 {
+		if c == '+' || c == '/' || c == '=' {
+			t.Errorf("Secret key contains non-URL-safe character: %c", c)
+		}
 	}
 
 	// Check uniqueness

weed/admin/dash/worker_grpc_server.go

@@ -108,6 +108,8 @@ func (s *WorkerGrpcServer) StartWithTLS(port int) error {
 	go s.cleanupRoutine()
 	go s.activeLogFetchLoop()
 
+	pb.ServeGrpcOnLocalSocket(grpcServer, port)
+
 	// Start serving in a goroutine
 	go func() {
 		if err := s.grpcServer.Serve(listener); err != nil {
@@ -785,15 +787,6 @@ func (s *WorkerGrpcServer) RequestTaskLogsFromAllWorkers(taskID string, maxEntri
 	return results, nil
 }
 
-// convertTaskParameters converts task parameters to protobuf format
-func convertTaskParameters(params map[string]interface{}) map[string]string {
-	result := make(map[string]string)
-	for key, value := range params {
-		result[key] = fmt.Sprintf("%v", value)
-	}
-	return result
-}
-
 func findClientAddress(ctx context.Context) string {
 	// fmt.Printf("FromContext %+v\n", ctx)
 	pr, ok := peer.FromContext(ctx)

weed/admin/handlers/cluster_handlers.go

@@ -312,29 +312,6 @@ func (h *ClusterHandlers) ShowClusterFilers(w http.ResponseWriter, r *http.Reque
 	}
 }
 
-// ShowClusterBrokers renders the cluster message brokers page
-func (h *ClusterHandlers) ShowClusterBrokers(w http.ResponseWriter, r *http.Request) {
-	// Get cluster brokers data
-	brokersData, err := h.adminServer.GetClusterBrokers()
-	if err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "Failed to get cluster brokers: "+err.Error())
-		return
-	}
-
-	username := usernameOrDefault(r)
-	brokersData.Username = username
-
-	// Render HTML template
-	w.Header().Set("Content-Type", "text/html")
-	brokersComponent := app.ClusterBrokers(*brokersData)
-	viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
-	layoutComponent := layout.Layout(viewCtx, brokersComponent)
-	if err := layoutComponent.Render(r.Context(), w); err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
-		return
-	}
-}
-
 // GetClusterTopology returns the cluster topology as JSON
 func (h *ClusterHandlers) GetClusterTopology(w http.ResponseWriter, r *http.Request) {
 	topology, err := h.adminServer.GetClusterTopology()

weed/admin/handlers/mq_handlers.go

@@ -78,34 +78,6 @@ func (h *MessageQueueHandlers) ShowTopics(w http.ResponseWriter, r *http.Request
 	}
 }
 
-// ShowSubscribers renders the message queue subscribers page
-func (h *MessageQueueHandlers) ShowSubscribers(w http.ResponseWriter, r *http.Request) {
-	// Get subscribers data
-	subscribersData, err := h.adminServer.GetSubscribers()
-	if err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "Failed to get subscribers: "+err.Error())
-		return
-	}
-
-	// Set username
-	username := dash.UsernameFromContext(r.Context())
-	if username == "" {
-		username = "admin"
-	}
-	subscribersData.Username = username
-
-	// Render HTML template
-	w.Header().Set("Content-Type", "text/html")
-	subscribersComponent := app.Subscribers(*subscribersData)
-	viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
-	layoutComponent := layout.Layout(viewCtx, subscribersComponent)
-	err = layoutComponent.Render(r.Context(), w)
-	if err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
-		return
-	}
-}
-
 // ShowTopicDetails renders the topic details page
 func (h *MessageQueueHandlers) ShowTopicDetails(w http.ResponseWriter, r *http.Request) {
 	// Get topic parameters from URL

weed/admin/handlers/user_handlers.go

@@ -93,6 +93,11 @@ func (h *UserHandlers) UpdateUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if h.adminServer.IsStaticUser(username) {
+		writeJSONError(w, http.StatusForbidden, "Cannot modify static user "+username+" (loaded from config file)")
+		return
+	}
+
 	var req dash.UpdateUserRequest
 	if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
 		writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
@@ -120,6 +125,11 @@ func (h *UserHandlers) DeleteUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if h.adminServer.IsStaticUser(username) {
+		writeJSONError(w, http.StatusForbidden, "Cannot delete static user "+username+" (loaded from config file)")
+		return
+	}
+
 	err := h.adminServer.DeleteObjectStoreUser(username)
 	if err != nil {
 		glog.Errorf("Failed to delete user %s: %v", username, err)

weed/admin/maintenance/config_verification.go

@@ -1,124 +0,0 @@
-package maintenance
-
-import (
-	"fmt"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
-)
-
-// VerifyProtobufConfig demonstrates that the protobuf configuration system is working
-func VerifyProtobufConfig() error {
-	// Create configuration manager
-	configManager := NewMaintenanceConfigManager()
-	config := configManager.GetConfig()
-
-	// Verify basic configuration
-	if !config.Enabled {
-		return fmt.Errorf("expected config to be enabled by default")
-	}
-
-	if config.ScanIntervalSeconds != 30*60 {
-		return fmt.Errorf("expected scan interval to be 1800 seconds, got %d", config.ScanIntervalSeconds)
-	}
-
-	// Verify policy configuration
-	if config.Policy == nil {
-		return fmt.Errorf("expected policy to be configured")
-	}
-
-	if config.Policy.GlobalMaxConcurrent != 4 {
-		return fmt.Errorf("expected global max concurrent to be 4, got %d", config.Policy.GlobalMaxConcurrent)
-	}
-
-	// Verify task policies
-	vacuumPolicy := config.Policy.TaskPolicies["vacuum"]
-	if vacuumPolicy == nil {
-		return fmt.Errorf("expected vacuum policy to be configured")
-	}
-
-	if !vacuumPolicy.Enabled {
-		return fmt.Errorf("expected vacuum policy to be enabled")
-	}
-
-	// Verify typed configuration access
-	vacuumConfig := vacuumPolicy.GetVacuumConfig()
-	if vacuumConfig == nil {
-		return fmt.Errorf("expected vacuum config to be accessible")
-	}
-
-	if vacuumConfig.GarbageThreshold != 0.3 {
-		return fmt.Errorf("expected garbage threshold to be 0.3, got %f", vacuumConfig.GarbageThreshold)
-	}
-
-	// Verify helper functions work
-	if !IsTaskEnabled(config.Policy, "vacuum") {
-		return fmt.Errorf("expected vacuum task to be enabled via helper function")
-	}
-
-	maxConcurrent := GetMaxConcurrent(config.Policy, "vacuum")
-	if maxConcurrent != 2 {
-		return fmt.Errorf("expected vacuum max concurrent to be 2, got %d", maxConcurrent)
-	}
-
-	// Verify erasure coding configuration
-	ecPolicy := config.Policy.TaskPolicies["erasure_coding"]
-	if ecPolicy == nil {
-		return fmt.Errorf("expected EC policy to be configured")
-	}
-
-	ecConfig := ecPolicy.GetErasureCodingConfig()
-	if ecConfig == nil {
-		return fmt.Errorf("expected EC config to be accessible")
-	}
-
-	// Verify configurable EC fields only
-	if ecConfig.FullnessRatio <= 0 || ecConfig.FullnessRatio > 1 {
-		return fmt.Errorf("expected EC config to have valid fullness ratio (0-1), got %f", ecConfig.FullnessRatio)
-	}
-
-	return nil
-}
-
-// GetProtobufConfigSummary returns a summary of the current protobuf configuration
-func GetProtobufConfigSummary() string {
-	configManager := NewMaintenanceConfigManager()
-	config := configManager.GetConfig()
-
-	summary := fmt.Sprintf("SeaweedFS Protobuf Maintenance Configuration:\n")
-	summary += fmt.Sprintf("  Enabled: %v\n", config.Enabled)
-	summary += fmt.Sprintf("  Scan Interval: %d seconds\n", config.ScanIntervalSeconds)
-	summary += fmt.Sprintf("  Max Retries: %d\n", config.MaxRetries)
-	summary += fmt.Sprintf("  Global Max Concurrent: %d\n", config.Policy.GlobalMaxConcurrent)
-	summary += fmt.Sprintf("  Task Policies: %d configured\n", len(config.Policy.TaskPolicies))
-
-	for taskType, policy := range config.Policy.TaskPolicies {
-		summary += fmt.Sprintf("    %s: enabled=%v, max_concurrent=%d\n",
-			taskType, policy.Enabled, policy.MaxConcurrent)
-	}
-
-	return summary
-}
-
-// CreateCustomConfig demonstrates creating a custom protobuf configuration
-func CreateCustomConfig() *worker_pb.MaintenanceConfig {
-	return &worker_pb.MaintenanceConfig{
-		Enabled:             true,
-		ScanIntervalSeconds: 60 * 60, // 1 hour
-		MaxRetries:          5,
-		Policy: &worker_pb.MaintenancePolicy{
-			GlobalMaxConcurrent: 8,
-			TaskPolicies: map[string]*worker_pb.TaskPolicy{
-				"custom_vacuum": {
-					Enabled:       true,
-					MaxConcurrent: 4,
-					TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
-						VacuumConfig: &worker_pb.VacuumTaskConfig{
-							GarbageThreshold:  0.5,
-							MinVolumeAgeHours: 48,
-						},
-					},
-				},
-			},
-		},
-	}
-}

weed/admin/maintenance/maintenance_config_proto.go

@@ -1,24 +1,9 @@
 package maintenance
 
 import (
-	"fmt"
-	"time"
-
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
 )
 
-// MaintenanceConfigManager handles protobuf-based configuration
-type MaintenanceConfigManager struct {
-	config *worker_pb.MaintenanceConfig
-}
-
-// NewMaintenanceConfigManager creates a new config manager with defaults
-func NewMaintenanceConfigManager() *MaintenanceConfigManager {
-	return &MaintenanceConfigManager{
-		config: DefaultMaintenanceConfigProto(),
-	}
-}
-
 // DefaultMaintenanceConfigProto returns default configuration as protobuf
 func DefaultMaintenanceConfigProto() *worker_pb.MaintenanceConfig {
 	return &worker_pb.MaintenanceConfig{
@@ -34,253 +19,3 @@ func DefaultMaintenanceConfigProto() *worker_pb.MaintenanceConfig {
 		Policy: nil,
 	}
 }
-
-// GetConfig returns the current configuration
-func (mcm *MaintenanceConfigManager) GetConfig() *worker_pb.MaintenanceConfig {
-	return mcm.config
-}
-
-// Type-safe configuration accessors
-
-// GetVacuumConfig returns vacuum-specific configuration for a task type
-func (mcm *MaintenanceConfigManager) GetVacuumConfig(taskType string) *worker_pb.VacuumTaskConfig {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		if vacuumConfig := policy.GetVacuumConfig(); vacuumConfig != nil {
-			return vacuumConfig
-		}
-	}
-	// Return defaults if not configured
-	return &worker_pb.VacuumTaskConfig{
-		GarbageThreshold:  0.3,
-		MinVolumeAgeHours: 24,
-	}
-}
-
-// GetErasureCodingConfig returns EC-specific configuration for a task type
-func (mcm *MaintenanceConfigManager) GetErasureCodingConfig(taskType string) *worker_pb.ErasureCodingTaskConfig {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		if ecConfig := policy.GetErasureCodingConfig(); ecConfig != nil {
-			return ecConfig
-		}
-	}
-	// Return defaults if not configured
-	return &worker_pb.ErasureCodingTaskConfig{
-		FullnessRatio:    0.95,
-		QuietForSeconds:  3600,
-		MinVolumeSizeMb:  100,
-		CollectionFilter: "",
-	}
-}
-
-// GetBalanceConfig returns balance-specific configuration for a task type
-func (mcm *MaintenanceConfigManager) GetBalanceConfig(taskType string) *worker_pb.BalanceTaskConfig {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		if balanceConfig := policy.GetBalanceConfig(); balanceConfig != nil {
-			return balanceConfig
-		}
-	}
-	// Return defaults if not configured
-	return &worker_pb.BalanceTaskConfig{
-		ImbalanceThreshold: 0.2,
-		MinServerCount:     2,
-	}
-}
-
-// GetReplicationConfig returns replication-specific configuration for a task type
-func (mcm *MaintenanceConfigManager) GetReplicationConfig(taskType string) *worker_pb.ReplicationTaskConfig {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		if replicationConfig := policy.GetReplicationConfig(); replicationConfig != nil {
-			return replicationConfig
-		}
-	}
-	// Return defaults if not configured
-	return &worker_pb.ReplicationTaskConfig{
-		TargetReplicaCount: 2,
-	}
-}
-
-// Typed convenience methods for getting task configurations
-
-// GetVacuumTaskConfigForType returns vacuum configuration for a specific task type
-func (mcm *MaintenanceConfigManager) GetVacuumTaskConfigForType(taskType string) *worker_pb.VacuumTaskConfig {
-	return GetVacuumTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
-}
-
-// GetErasureCodingTaskConfigForType returns erasure coding configuration for a specific task type
-func (mcm *MaintenanceConfigManager) GetErasureCodingTaskConfigForType(taskType string) *worker_pb.ErasureCodingTaskConfig {
-	return GetErasureCodingTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
-}
-
-// GetBalanceTaskConfigForType returns balance configuration for a specific task type
-func (mcm *MaintenanceConfigManager) GetBalanceTaskConfigForType(taskType string) *worker_pb.BalanceTaskConfig {
-	return GetBalanceTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
-}
-
-// GetReplicationTaskConfigForType returns replication configuration for a specific task type
-func (mcm *MaintenanceConfigManager) GetReplicationTaskConfigForType(taskType string) *worker_pb.ReplicationTaskConfig {
-	return GetReplicationTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
-}
-
-// Helper methods
-
-func (mcm *MaintenanceConfigManager) getTaskPolicy(taskType string) *worker_pb.TaskPolicy {
-	if mcm.config.Policy != nil && mcm.config.Policy.TaskPolicies != nil {
-		return mcm.config.Policy.TaskPolicies[taskType]
-	}
-	return nil
-}
-
-// IsTaskEnabled returns whether a task type is enabled
-func (mcm *MaintenanceConfigManager) IsTaskEnabled(taskType string) bool {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		return policy.Enabled
-	}
-	return false
-}
-
-// GetMaxConcurrent returns the max concurrent limit for a task type
-func (mcm *MaintenanceConfigManager) GetMaxConcurrent(taskType string) int32 {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		return policy.MaxConcurrent
-	}
-	return 1 // Default
-}
-
-// GetRepeatInterval returns the repeat interval for a task type in seconds
-func (mcm *MaintenanceConfigManager) GetRepeatInterval(taskType string) int32 {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		return policy.RepeatIntervalSeconds
-	}
-	return mcm.config.Policy.DefaultRepeatIntervalSeconds
-}
-
-// GetCheckInterval returns the check interval for a task type in seconds
-func (mcm *MaintenanceConfigManager) GetCheckInterval(taskType string) int32 {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		return policy.CheckIntervalSeconds
-	}
-	return mcm.config.Policy.DefaultCheckIntervalSeconds
-}
-
-// Duration accessor methods
-
-// GetScanInterval returns the scan interval as a time.Duration
-func (mcm *MaintenanceConfigManager) GetScanInterval() time.Duration {
-	return time.Duration(mcm.config.ScanIntervalSeconds) * time.Second
-}
-
-// GetWorkerTimeout returns the worker timeout as a time.Duration
-func (mcm *MaintenanceConfigManager) GetWorkerTimeout() time.Duration {
-	return time.Duration(mcm.config.WorkerTimeoutSeconds) * time.Second
-}
-
-// GetTaskTimeout returns the task timeout as a time.Duration
-func (mcm *MaintenanceConfigManager) GetTaskTimeout() time.Duration {
-	return time.Duration(mcm.config.TaskTimeoutSeconds) * time.Second
-}
-
-// GetRetryDelay returns the retry delay as a time.Duration
-func (mcm *MaintenanceConfigManager) GetRetryDelay() time.Duration {
-	return time.Duration(mcm.config.RetryDelaySeconds) * time.Second
-}
-
-// GetCleanupInterval returns the cleanup interval as a time.Duration
-func (mcm *MaintenanceConfigManager) GetCleanupInterval() time.Duration {
-	return time.Duration(mcm.config.CleanupIntervalSeconds) * time.Second
-}
-
-// GetTaskRetention returns the task retention period as a time.Duration
-func (mcm *MaintenanceConfigManager) GetTaskRetention() time.Duration {
-	return time.Duration(mcm.config.TaskRetentionSeconds) * time.Second
-}
-
-// ValidateMaintenanceConfigWithSchema validates protobuf maintenance configuration using ConfigField rules
-func ValidateMaintenanceConfigWithSchema(config *worker_pb.MaintenanceConfig) error {
-	if config == nil {
-		return fmt.Errorf("configuration cannot be nil")
-	}
-
-	// Get the schema to access field validation rules
-	schema := GetMaintenanceConfigSchema()
-
-	// Validate each field individually using the ConfigField rules
-	if err := validateFieldWithSchema(schema, "enabled", config.Enabled); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "scan_interval_seconds", int(config.ScanIntervalSeconds)); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "worker_timeout_seconds", int(config.WorkerTimeoutSeconds)); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "task_timeout_seconds", int(config.TaskTimeoutSeconds)); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "retry_delay_seconds", int(config.RetryDelaySeconds)); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "max_retries", int(config.MaxRetries)); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "cleanup_interval_seconds", int(config.CleanupIntervalSeconds)); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "task_retention_seconds", int(config.TaskRetentionSeconds)); err != nil {
-		return err
-	}
-
-	// Validate policy fields if present
-	if config.Policy != nil {
-		// Note: These field names might need to be adjusted based on the actual schema
-		if err := validatePolicyField("global_max_concurrent", int(config.Policy.GlobalMaxConcurrent)); err != nil {
-			return err
-		}
-
-		if err := validatePolicyField("default_repeat_interval_seconds", int(config.Policy.DefaultRepeatIntervalSeconds)); err != nil {
-			return err
-		}
-
-		if err := validatePolicyField("default_check_interval_seconds", int(config.Policy.DefaultCheckIntervalSeconds)); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// validateFieldWithSchema validates a single field using its ConfigField definition
-func validateFieldWithSchema(schema *MaintenanceConfigSchema, fieldName string, value interface{}) error {
-	field := schema.GetFieldByName(fieldName)
-	if field == nil {
-		// Field not in schema, skip validation
-		return nil
-	}
-
-	return field.ValidateValue(value)
-}
-
-// validatePolicyField validates policy fields (simplified validation for now)
-func validatePolicyField(fieldName string, value int) error {
-	switch fieldName {
-	case "global_max_concurrent":
-		if value < 1 || value > 20 {
-			return fmt.Errorf("Global Max Concurrent must be between 1 and 20, got %d", value)
-		}
-	case "default_repeat_interval":
-		if value < 1 || value > 168 {
-			return fmt.Errorf("Default Repeat Interval must be between 1 and 168 hours, got %d", value)
-		}
-	case "default_check_interval":
-		if value < 1 || value > 168 {
-			return fmt.Errorf("Default Check Interval must be between 1 and 168 hours, got %d", value)
-		}
-	}
-	return nil
-}

weed/admin/maintenance/maintenance_queue.go

@@ -33,80 +33,16 @@ func (mq *MaintenanceQueue) SetPersistence(persistence TaskPersistence) {
 	glog.V(1).Infof("Maintenance queue configured with task persistence")
 }
 
-// LoadTasksFromPersistence loads tasks from persistent storage on startup
+// LoadTasksFromPersistence is called on startup. Previous task states are NOT loaded
+// into memory — the maintenance scanner will re-detect current needs from the live
+// cluster state. Stale task files from previous runs are deleted from disk.
 func (mq *MaintenanceQueue) LoadTasksFromPersistence() error {
-	if mq.persistence == nil {
-		glog.V(1).Infof("No task persistence configured, skipping task loading")
-		return nil
-	}
-
-	mq.mutex.Lock()
-	defer mq.mutex.Unlock()
-
-	glog.Infof("Loading tasks from persistence...")
-
-	tasks, err := mq.persistence.LoadAllTaskStates()
-	if err != nil {
-		return fmt.Errorf("failed to load task states: %w", err)
-	}
-
-	glog.Infof("DEBUG LoadTasksFromPersistence: Found %d tasks in persistence", len(tasks))
-
-	// Reset task maps
-	mq.tasks = make(map[string]*MaintenanceTask)
-	mq.pendingTasks = make([]*MaintenanceTask, 0)
-
-	// Load tasks by status
-	for _, task := range tasks {
-		glog.Infof("DEBUG LoadTasksFromPersistence: Loading task %s (type: %s, status: %s, scheduled: %v)", task.ID, task.Type, task.Status, task.ScheduledAt)
-		mq.tasks[task.ID] = task
-
-		switch task.Status {
-		case TaskStatusPending:
-			glog.Infof("DEBUG LoadTasksFromPersistence: Adding task %s to pending queue", task.ID)
-			mq.pendingTasks = append(mq.pendingTasks, task)
-		case TaskStatusAssigned, TaskStatusInProgress:
-			// For assigned/in-progress tasks, we need to check if the worker is still available
-			// If not, we should fail them and make them eligible for retry
-			if task.WorkerID != "" {
-				if _, exists := mq.workers[task.WorkerID]; !exists {
-					glog.Warningf("Task %s was assigned to unavailable worker %s, marking as failed", task.ID, task.WorkerID)
-					task.Status = TaskStatusFailed
-					task.Error = "Worker unavailable after restart"
-					completedTime := time.Now()
-					task.CompletedAt = &completedTime
-
-					// Check if it should be retried
-					if task.RetryCount < task.MaxRetries {
-						task.RetryCount++
-						task.Status = TaskStatusPending
-						task.WorkerID = ""
-						task.StartedAt = nil
-						task.CompletedAt = nil
-						task.Error = ""
-						task.ScheduledAt = time.Now().Add(1 * time.Minute) // Retry after restart delay
-						glog.Infof("DEBUG LoadTasksFromPersistence: Retrying task %s, adding to pending queue", task.ID)
-						mq.pendingTasks = append(mq.pendingTasks, task)
-					}
-				}
-			}
-		}
-
-		// Sync task with ActiveTopology for capacity tracking
-		if mq.integration != nil {
-			mq.integration.SyncTask(task)
+	if mq.persistence != nil {
+		if err := mq.persistence.DeleteAllTaskStates(); err != nil {
+			glog.Warningf("Failed to clean up old task files: %v", err)
 		}
 	}
-
-	// Sort pending tasks by priority and schedule time
-	sort.Slice(mq.pendingTasks, func(i, j int) bool {
-		if mq.pendingTasks[i].Priority != mq.pendingTasks[j].Priority {
-			return mq.pendingTasks[i].Priority > mq.pendingTasks[j].Priority
-		}
-		return mq.pendingTasks[i].ScheduledAt.Before(mq.pendingTasks[j].ScheduledAt)
-	})
-
-	glog.Infof("Loaded %d tasks from persistence (%d pending)", len(tasks), len(mq.pendingTasks))
+	glog.Infof("Task queue initialized (previous tasks will be re-detected by scanner)")
 	return nil
 }
 
@@ -119,6 +55,14 @@ func (mq *MaintenanceQueue) saveTaskState(task *MaintenanceTask) {
 	}
 }
 
+func (mq *MaintenanceQueue) deleteTaskState(taskID string) {
+	if mq.persistence != nil {
+		if err := mq.persistence.DeleteTaskState(taskID); err != nil {
+			glog.V(2).Infof("Failed to delete task state for %s: %v", taskID, err)
+		}
+	}
+}
+
 // cleanupCompletedTasks removes old completed tasks beyond the retention limit
 func (mq *MaintenanceQueue) cleanupCompletedTasks() {
 	if mq.persistence != nil {
@@ -128,10 +72,24 @@ func (mq *MaintenanceQueue) cleanupCompletedTasks() {
 	}
 }
 
+const MaxTasksPerType = 100
+
 // AddTask adds a new maintenance task to the queue with deduplication
 func (mq *MaintenanceQueue) AddTask(task *MaintenanceTask) {
 	mq.mutex.Lock()
 
+	// Enforce per-type capacity limit (only counting active tasks)
+	if mq.countActiveTasksByType(task.Type) >= MaxTasksPerType {
+		// Purge terminal tasks first, then recheck
+		mq.purgeTerminalTasksLocked()
+		if mq.countActiveTasksByType(task.Type) >= MaxTasksPerType {
+			mq.mutex.Unlock()
+			glog.V(1).Infof("Task skipped (type %s at capacity %d): volume %d on %s",
+				task.Type, MaxTasksPerType, task.VolumeID, task.Server)
+			return
+		}
+	}
+
 	// Enforce one queued/active task per volume (across all task types).
 	if mq.hasQueuedOrActiveTaskForVolume(task.VolumeID) {
 		mq.mutex.Unlock()
@@ -197,6 +155,30 @@ func (mq *MaintenanceQueue) AddTask(task *MaintenanceTask) {
 		taskSnapshot.ID, taskSnapshot.Type, taskSnapshot.VolumeID, taskSnapshot.Server, taskSnapshot.Priority, scheduleInfo, taskSnapshot.Reason)
 }
 
+// countActiveTasksByType returns the number of active (non-terminal) tasks of a given type. Caller must hold mq.mutex.
+func (mq *MaintenanceQueue) countActiveTasksByType(taskType MaintenanceTaskType) int {
+	count := 0
+	for _, t := range mq.tasks {
+		if t.Type == taskType {
+			switch t.Status {
+			case TaskStatusPending, TaskStatusAssigned, TaskStatusInProgress:
+				count++
+			}
+		}
+	}
+	return count
+}
+
+// purgeTerminalTasksLocked removes terminal tasks from the in-memory map. Caller must hold mq.mutex.
+func (mq *MaintenanceQueue) purgeTerminalTasksLocked() {
+	for id, task := range mq.tasks {
+		switch task.Status {
+		case TaskStatusCompleted, TaskStatusFailed, TaskStatusCancelled:
+			delete(mq.tasks, id)
+		}
+	}
+}
+
 // hasQueuedOrActiveTaskForVolume checks if any pending/assigned/in-progress task already exists for this volume.
 // Caller must hold mq.mutex.
 func (mq *MaintenanceQueue) hasQueuedOrActiveTaskForVolume(volumeID uint32) bool {
@@ -273,6 +255,9 @@ func (mq *MaintenanceQueue) CancelPendingTasksByType(taskType MaintenanceTaskTyp
 
 // AddTasksFromResults converts detection results to tasks and adds them to the queue
 func (mq *MaintenanceQueue) AddTasksFromResults(results []*TaskDetectionResult) {
+	// Purge terminal tasks from memory before adding new ones
+	mq.purgeTerminalTasks()
+
 	for _, result := range results {
 		// Validate that task has proper typed parameters
 		if result.TypedParams == nil {
@@ -297,6 +282,21 @@ func (mq *MaintenanceQueue) AddTasksFromResults(results []*TaskDetectionResult)
 	}
 }
 
+// purgeTerminalTasks removes completed/failed/cancelled tasks from memory.
+// Terminal tasks are already deleted from disk by CompleteTask, so this
+// only needs to clean up the in-memory map.
+func (mq *MaintenanceQueue) purgeTerminalTasks() {
+	mq.mutex.Lock()
+	before := len(mq.tasks)
+	mq.purgeTerminalTasksLocked()
+	purged := before - len(mq.tasks)
+	mq.mutex.Unlock()
+
+	if purged > 0 {
+		glog.V(1).Infof("Purged %d terminal tasks from memory", purged)
+	}
+}
+
 // GetNextTask returns the next available task for a worker
 func (mq *MaintenanceQueue) GetNextTask(workerID string, capabilities []MaintenanceTaskType) *MaintenanceTask {
 	// Use read lock for initial checks and search
@@ -570,7 +570,6 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 		}
 	}
 	taskStatus := task.Status
-	taskCount := len(mq.tasks)
 	// Snapshot task state while lock is still held to avoid data race
 	var taskToSaveSnapshot *MaintenanceTask
 	if taskToSave != nil {
@@ -578,9 +577,18 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 	}
 	mq.mutex.Unlock()
 
-	// Save task state to persistence outside the lock
+	// Only persist non-terminal tasks (retries). Completed/failed tasks stay
+	// in memory for the UI but are not written to disk — they would just
+	// accumulate and slow down future startups.
 	if taskToSaveSnapshot != nil {
-		mq.saveTaskState(taskToSaveSnapshot)
+		switch taskStatus {
+		case TaskStatusPending:
+			// Retry — save so the task survives a restart
+			mq.saveTaskState(taskToSaveSnapshot)
+		case TaskStatusCompleted, TaskStatusFailed, TaskStatusCancelled:
+			// Terminal — delete the file if one exists from a previous state
+			mq.deleteTaskState(taskToSaveSnapshot.ID)
+		}
 	}
 
 	if logFn != nil {
@@ -591,13 +599,6 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 	if taskStatus != TaskStatusPending {
 		mq.removePendingOperation(taskID)
 	}
-
-	// Periodically cleanup old completed tasks (when total task count is a multiple of 10)
-	if taskStatus == TaskStatusCompleted {
-		if taskCount%10 == 0 {
-			go mq.cleanupCompletedTasks()
-		}
-	}
 }
 
 // isNonRetriableError returns true for errors that will never succeed on retry,
@@ -1055,28 +1056,6 @@ func (mq *MaintenanceQueue) getMaxConcurrentForTaskType(taskType MaintenanceTask
 	return 1
 }
 
-// getRunningTasks returns all currently running tasks
-func (mq *MaintenanceQueue) getRunningTasks() []*MaintenanceTask {
-	var runningTasks []*MaintenanceTask
-	for _, task := range mq.tasks {
-		if task.Status == TaskStatusAssigned || task.Status == TaskStatusInProgress {
-			runningTasks = append(runningTasks, task)
-		}
-	}
-	return runningTasks
-}
-
-// getAvailableWorkers returns all workers that can take more work
-func (mq *MaintenanceQueue) getAvailableWorkers() []*MaintenanceWorker {
-	var availableWorkers []*MaintenanceWorker
-	for _, worker := range mq.workers {
-		if worker.Status == "active" && worker.CurrentLoad < worker.MaxConcurrent {
-			availableWorkers = append(availableWorkers, worker)
-		}
-	}
-	return availableWorkers
-}
-
 // trackPendingOperation adds a task to the pending operations tracker
 func (mq *MaintenanceQueue) trackPendingOperation(task *MaintenanceTask) {
 	if mq.integration == nil {

weed/admin/maintenance/maintenance_queue_test.go

@@ -693,10 +693,11 @@ func (m *MockPersistence) SaveTaskState(task *MaintenanceTask) error
 func (m *MockPersistence) LoadTaskState(taskID string) (*MaintenanceTask, error)    { return nil, nil }
 func (m *MockPersistence) LoadAllTaskStates() ([]*MaintenanceTask, error)           { return m.tasks, nil }
 func (m *MockPersistence) DeleteTaskState(taskID string) error                      { return nil }
+func (m *MockPersistence) DeleteAllTaskStates() error                               { return nil }
 func (m *MockPersistence) CleanupCompletedTasks() error                             { return nil }
 func (m *MockPersistence) SaveTaskPolicy(taskType string, policy *TaskPolicy) error { return nil }
 
-func TestMaintenanceQueue_LoadTasksCapacitySync(t *testing.T) {
+func TestMaintenanceQueue_LoadTasksStartsEmpty(t *testing.T) {
 	// Setup
 	policy := &MaintenancePolicy{
 		TaskPolicies: map[string]*worker_pb.TaskPolicy{
@@ -704,56 +705,25 @@ func TestMaintenanceQueue_LoadTasksCapacitySync(t *testing.T) {
 		},
 	}
 	mq := NewMaintenanceQueue(policy)
-	integration := NewMaintenanceIntegration(mq, policy)
-	mq.SetIntegration(integration)
-	at := integration.GetActiveTopology()
 
-	topologyInfo := &master_pb.TopologyInfo{
-		DataCenterInfos: []*master_pb.DataCenterInfo{
-			{
-				Id: "dc1",
-				RackInfos: []*master_pb.RackInfo{
-					{
-						Id: "rack1",
-						DataNodeInfos: []*master_pb.DataNodeInfo{
-							{
-								Id: "server1",
-								DiskInfos: map[string]*master_pb.DiskInfo{
-									"hdd1": {DiskId: 1, VolumeCount: 1, MaxVolumeCount: 10},
-									"hdd2": {DiskId: 2, VolumeCount: 0, MaxVolumeCount: 10},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-	at.UpdateTopology(topologyInfo)
-
-	// Setup mock persistence with a pending task
-	taskID := "load_test_123"
+	// Setup mock persistence with tasks — these should NOT be loaded
 	mockTask := &MaintenanceTask{
-		ID:     taskID,
+		ID:     "old_task_123",
 		Type:   "balance",
 		Status: TaskStatusPending,
-		TypedParams: &worker_pb.TaskParams{
-			TaskId:  taskID,
-			Sources: []*worker_pb.TaskSource{{Node: "server1", DiskId: 1}},
-			Targets: []*worker_pb.TaskTarget{{Node: "server1", DiskId: 2}},
-		},
 	}
 	mq.SetPersistence(&MockPersistence{tasks: []*MaintenanceTask{mockTask}})
 
-	// Load tasks
+	// LoadTasksFromPersistence should be a no-op — scanner will re-detect
 	err := mq.LoadTasksFromPersistence()
 	if err != nil {
-		t.Fatalf("Failed to load tasks: %v", err)
+		t.Fatalf("LoadTasksFromPersistence failed: %v", err)
 	}
 
-	// Verify capacity is reserved in ActiveTopology after loading (9 left)
-	if at.GetEffectiveAvailableCapacity("server1", 2) != 9 {
-		t.Errorf("Expected capacity 9 after loading tasks, got %d", at.GetEffectiveAvailableCapacity("server1", 2))
+	// Queue should be empty — tasks will be re-detected by scanner
+	stats := mq.GetStats()
+	if stats.TotalTasks != 0 {
+		t.Errorf("Expected 0 tasks after startup, got %d", stats.TotalTasks)
 	}
 }
 

weed/admin/maintenance/maintenance_scanner.go

@@ -32,21 +32,18 @@ func NewMaintenanceScanner(adminClient AdminClient, policy *MaintenancePolicy, q
 
 // ScanForMaintenanceTasks analyzes the cluster and generates maintenance tasks
 func (ms *MaintenanceScanner) ScanForMaintenanceTasks() ([]*TaskDetectionResult, error) {
-	// Get volume health metrics
-	volumeMetrics, err := ms.getVolumeHealthMetrics()
+	// Get volume health metrics directly in task-system format, along with topology info
+	taskMetrics, topologyInfo, err := ms.getVolumeHealthMetrics()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get volume health metrics: %w", err)
 	}
 
 	// Use task system for all task types
 	if ms.integration != nil {
-		// Convert metrics to task system format
-		taskMetrics := ms.convertToTaskMetrics(volumeMetrics)
-
 		// Update topology information for complete cluster view (including empty servers)
 		// This must happen before task detection to ensure EC placement can consider all servers
-		if ms.lastTopologyInfo != nil {
-			if err := ms.integration.UpdateTopologyInfo(ms.lastTopologyInfo); err != nil {
+		if topologyInfo != nil {
+			if err := ms.integration.UpdateTopologyInfo(topologyInfo); err != nil {
 				glog.Errorf("Failed to update topology info for empty servers: %v", err)
 				// Don't fail the scan - continue with just volume-bearing servers
 			} else {
@@ -70,9 +67,12 @@ func (ms *MaintenanceScanner) ScanForMaintenanceTasks() ([]*TaskDetectionResult,
 	return []*TaskDetectionResult{}, nil
 }
 
-// getVolumeHealthMetrics collects health information for all volumes
-func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*VolumeHealthMetrics, error) {
-	var metrics []*VolumeHealthMetrics
+// getVolumeHealthMetrics collects health information for all volumes.
+// Returns metrics in task-system format directly (no intermediate copy) and
+// the topology info for updating the active topology.
+func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*types.VolumeHealthMetrics, *master_pb.TopologyInfo, error) {
+	var metrics []*types.VolumeHealthMetrics
+	var topologyInfo *master_pb.TopologyInfo
 
 	glog.V(1).Infof("Collecting volume health metrics from master")
 	err := ms.adminClient.WithMasterClient(func(client master_pb.SeaweedClient) error {
@@ -89,30 +89,28 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*VolumeHealthMetrics,
 
 		volumeSizeLimitBytes := uint64(resp.VolumeSizeLimitMb) * 1024 * 1024 // Convert MB to bytes
 
-		// Track all nodes discovered in topology
-		var allNodesInTopology []string
-		var nodesWithVolumes []string
-		var nodesWithoutVolumes []string
+		// Track node counts for summary logging (avoid accumulating full ID slices)
+		var totalNodes, nodesWithVolumes, nodesWithoutVolumes int
 
 		for _, dc := range resp.TopologyInfo.DataCenterInfos {
-			glog.V(2).Infof("Processing datacenter: %s", dc.Id)
+			glog.V(3).Infof("Processing datacenter: %s", dc.Id)
 			for _, rack := range dc.RackInfos {
-				glog.V(2).Infof("Processing rack: %s in datacenter: %s", rack.Id, dc.Id)
+				glog.V(3).Infof("Processing rack: %s in datacenter: %s", rack.Id, dc.Id)
 				for _, node := range rack.DataNodeInfos {
-					allNodesInTopology = append(allNodesInTopology, node.Id)
-					glog.V(2).Infof("Found volume server in topology: %s (disks: %d)", node.Id, len(node.DiskInfos))
+					totalNodes++
+					glog.V(3).Infof("Found volume server in topology: %s (disks: %d)", node.Id, len(node.DiskInfos))
 
 					hasVolumes := false
 					// Process each disk on this node
 					for diskType, diskInfo := range node.DiskInfos {
 						if len(diskInfo.VolumeInfos) > 0 {
 							hasVolumes = true
-							glog.V(2).Infof("Volume server %s disk %s has %d volumes", node.Id, diskType, len(diskInfo.VolumeInfos))
+							glog.V(3).Infof("Volume server %s disk %s has %d volumes", node.Id, diskType, len(diskInfo.VolumeInfos))
 						}
 
 						// Process volumes on this specific disk
 						for _, volInfo := range diskInfo.VolumeInfos {
-							metric := &VolumeHealthMetrics{
+							metric := &types.VolumeHealthMetrics{
 								VolumeID:         volInfo.Id,
 								Server:           node.Id,
 								ServerAddress:    node.Address,
@@ -138,7 +136,7 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*VolumeHealthMetrics,
 							}
 							metric.Age = time.Since(metric.LastModified)
 
-							glog.V(3).Infof("Volume %d on %s:%s (ID %d): size=%d, limit=%d, fullness=%.2f",
+							glog.V(4).Infof("Volume %d on %s:%s (ID %d): size=%d, limit=%d, fullness=%.2f",
 								metric.VolumeID, metric.Server, metric.DiskType, metric.DiskId, metric.Size, volumeSizeLimitBytes, metric.FullnessRatio)
 
 							metrics = append(metrics, metric)
@@ -146,29 +144,27 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*VolumeHealthMetrics,
 					}
 
 					if hasVolumes {
-						nodesWithVolumes = append(nodesWithVolumes, node.Id)
+						nodesWithVolumes++
 					} else {
-						nodesWithoutVolumes = append(nodesWithoutVolumes, node.Id)
+						nodesWithoutVolumes++
 						glog.V(1).Infof("Volume server %s found in topology but has no volumes", node.Id)
 					}
 				}
 			}
 		}
 
-		glog.Infof("Topology discovery complete:")
-		glog.Infof("  - Total volume servers in topology: %d (%v)", len(allNodesInTopology), allNodesInTopology)
-		glog.Infof("  - Volume servers with volumes: %d (%v)", len(nodesWithVolumes), nodesWithVolumes)
-		glog.Infof("  - Volume servers without volumes: %d (%v)", len(nodesWithoutVolumes), nodesWithoutVolumes)
+		glog.Infof("Topology discovery: %d volume servers (%d with volumes, %d without)",
+			totalNodes, nodesWithVolumes, nodesWithoutVolumes)
 
-		// Store topology info for volume shard tracker
-		ms.lastTopologyInfo = resp.TopologyInfo
+		// Return topology info as a local value (not retained on the scanner struct)
+		topologyInfo = resp.TopologyInfo
 
 		return nil
 	})
 
 	if err != nil {
 		glog.Errorf("Failed to get volume health metrics: %v", err)
-		return nil, err
+		return nil, nil, err
 	}
 
 	glog.V(1).Infof("Successfully collected metrics for %d actual volumes with disk ID information", len(metrics))
@@ -176,13 +172,13 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*VolumeHealthMetrics,
 	// Count actual replicas and identify EC volumes
 	ms.enrichVolumeMetrics(metrics)
 
-	return metrics, nil
+	return metrics, topologyInfo, nil
 }
 
 // enrichVolumeMetrics adds additional information like replica counts
-func (ms *MaintenanceScanner) enrichVolumeMetrics(metrics []*VolumeHealthMetrics) {
+func (ms *MaintenanceScanner) enrichVolumeMetrics(metrics []*types.VolumeHealthMetrics) {
 	// Group volumes by ID to count replicas
-	volumeGroups := make(map[uint32][]*VolumeHealthMetrics)
+	volumeGroups := make(map[uint32][]*types.VolumeHealthMetrics)
 	for _, metric := range metrics {
 		volumeGroups[metric.VolumeID] = append(volumeGroups[metric.VolumeID], metric)
 	}
@@ -193,41 +189,9 @@ func (ms *MaintenanceScanner) enrichVolumeMetrics(metrics []*VolumeHealthMetrics
 		for _, replica := range replicas {
 			replica.ReplicaCount = replicaCount
 		}
-		glog.V(3).Infof("Volume %d has %d replicas", volumeID, replicaCount)
+		glog.V(4).Infof("Volume %d has %d replicas", volumeID, replicaCount)
 	}
 
 	// TODO: Identify EC volumes by checking volume structure
 	// This would require querying volume servers for EC shard information
 }
-
-// convertToTaskMetrics converts existing volume metrics to task system format
-func (ms *MaintenanceScanner) convertToTaskMetrics(metrics []*VolumeHealthMetrics) []*types.VolumeHealthMetrics {
-	var simplified []*types.VolumeHealthMetrics
-
-	for _, metric := range metrics {
-		simplified = append(simplified, &types.VolumeHealthMetrics{
-			VolumeID:         metric.VolumeID,
-			Server:           metric.Server,
-			ServerAddress:    metric.ServerAddress,
-			DiskType:         metric.DiskType,
-			DiskId:           metric.DiskId,
-			DataCenter:       metric.DataCenter,
-			Rack:             metric.Rack,
-			Collection:       metric.Collection,
-			Size:             metric.Size,
-			DeletedBytes:     metric.DeletedBytes,
-			GarbageRatio:     metric.GarbageRatio,
-			LastModified:     metric.LastModified,
-			Age:              metric.Age,
-			ReplicaCount:     metric.ReplicaCount,
-			ExpectedReplicas: metric.ExpectedReplicas,
-			IsReadOnly:       metric.IsReadOnly,
-			HasRemoteCopy:    metric.HasRemoteCopy,
-			IsECVolume:       metric.IsECVolume,
-			FullnessRatio:    metric.FullnessRatio,
-		})
-	}
-
-	glog.V(2).Infof("Converted %d volume metrics with disk ID information for task detection", len(simplified))
-	return simplified
-}

weed/admin/maintenance/maintenance_types.go

@@ -2,15 +2,11 @@ package maintenance
 
 import (
 	"html/template"
-	"sort"
 	"sync"
 	"time"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
-	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
-	"github.com/seaweedfs/seaweedfs/weed/worker/types"
 )
 
 // AdminClient interface defines what the maintenance system needs from the admin server
@@ -21,51 +17,6 @@ type AdminClient interface {
 // MaintenanceTaskType represents different types of maintenance operations
 type MaintenanceTaskType string
 
-// GetRegisteredMaintenanceTaskTypes returns all registered task types as MaintenanceTaskType values
-// sorted alphabetically for consistent menu ordering
-func GetRegisteredMaintenanceTaskTypes() []MaintenanceTaskType {
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-	var taskTypes []MaintenanceTaskType
-
-	for workerTaskType := range typesRegistry.GetAllDetectors() {
-		maintenanceTaskType := MaintenanceTaskType(string(workerTaskType))
-		taskTypes = append(taskTypes, maintenanceTaskType)
-	}
-
-	// Sort task types alphabetically to ensure consistent menu ordering
-	sort.Slice(taskTypes, func(i, j int) bool {
-		return string(taskTypes[i]) < string(taskTypes[j])
-	})
-
-	return taskTypes
-}
-
-// GetMaintenanceTaskType returns a specific task type if it's registered, or empty string if not found
-func GetMaintenanceTaskType(taskTypeName string) MaintenanceTaskType {
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-
-	for workerTaskType := range typesRegistry.GetAllDetectors() {
-		if string(workerTaskType) == taskTypeName {
-			return MaintenanceTaskType(taskTypeName)
-		}
-	}
-
-	return MaintenanceTaskType("")
-}
-
-// IsMaintenanceTaskTypeRegistered checks if a task type is registered
-func IsMaintenanceTaskTypeRegistered(taskType MaintenanceTaskType) bool {
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-
-	for workerTaskType := range typesRegistry.GetAllDetectors() {
-		if string(workerTaskType) == string(taskType) {
-			return true
-		}
-	}
-
-	return false
-}
-
 // MaintenanceTaskPriority represents task execution priority
 type MaintenanceTaskPriority int
 
@@ -179,6 +130,7 @@ type TaskPersistence interface {
 	LoadTaskState(taskID string) (*MaintenanceTask, error)
 	LoadAllTaskStates() ([]*MaintenanceTask, error)
 	DeleteTaskState(taskID string) error
+	DeleteAllTaskStates() error
 	CleanupCompletedTasks() error
 
 	// Policy persistence
@@ -200,14 +152,6 @@ func GetTaskPolicy(mp *MaintenancePolicy, taskType MaintenanceTaskType) *TaskPol
 	return mp.TaskPolicies[string(taskType)]
 }
 
-// SetTaskPolicy sets the policy for a specific task type
-func SetTaskPolicy(mp *MaintenancePolicy, taskType MaintenanceTaskType, policy *TaskPolicy) {
-	if mp.TaskPolicies == nil {
-		mp.TaskPolicies = make(map[string]*TaskPolicy)
-	}
-	mp.TaskPolicies[string(taskType)] = policy
-}
-
 // IsTaskEnabled returns whether a task type is enabled
 func IsTaskEnabled(mp *MaintenancePolicy, taskType MaintenanceTaskType) bool {
 	policy := GetTaskPolicy(mp, taskType)
@@ -235,84 +179,6 @@ func GetRepeatInterval(mp *MaintenancePolicy, taskType MaintenanceTaskType) int
 	return int(policy.RepeatIntervalSeconds)
 }
 
-// GetVacuumTaskConfig returns the vacuum task configuration
-func GetVacuumTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.VacuumTaskConfig {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy == nil {
-		return nil
-	}
-	return policy.GetVacuumConfig()
-}
-
-// GetErasureCodingTaskConfig returns the erasure coding task configuration
-func GetErasureCodingTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.ErasureCodingTaskConfig {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy == nil {
-		return nil
-	}
-	return policy.GetErasureCodingConfig()
-}
-
-// GetBalanceTaskConfig returns the balance task configuration
-func GetBalanceTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.BalanceTaskConfig {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy == nil {
-		return nil
-	}
-	return policy.GetBalanceConfig()
-}
-
-// GetReplicationTaskConfig returns the replication task configuration
-func GetReplicationTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.ReplicationTaskConfig {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy == nil {
-		return nil
-	}
-	return policy.GetReplicationConfig()
-}
-
-// Note: GetTaskConfig was removed - use typed getters: GetVacuumTaskConfig, GetErasureCodingTaskConfig, GetBalanceTaskConfig, or GetReplicationTaskConfig
-
-// SetVacuumTaskConfig sets the vacuum task configuration
-func SetVacuumTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.VacuumTaskConfig) {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy != nil {
-		policy.TaskConfig = &worker_pb.TaskPolicy_VacuumConfig{
-			VacuumConfig: config,
-		}
-	}
-}
-
-// SetErasureCodingTaskConfig sets the erasure coding task configuration
-func SetErasureCodingTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.ErasureCodingTaskConfig) {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy != nil {
-		policy.TaskConfig = &worker_pb.TaskPolicy_ErasureCodingConfig{
-			ErasureCodingConfig: config,
-		}
-	}
-}
-
-// SetBalanceTaskConfig sets the balance task configuration
-func SetBalanceTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.BalanceTaskConfig) {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy != nil {
-		policy.TaskConfig = &worker_pb.TaskPolicy_BalanceConfig{
-			BalanceConfig: config,
-		}
-	}
-}
-
-// SetReplicationTaskConfig sets the replication task configuration
-func SetReplicationTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.ReplicationTaskConfig) {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy != nil {
-		policy.TaskConfig = &worker_pb.TaskPolicy_ReplicationConfig{
-			ReplicationConfig: config,
-		}
-	}
-}
-
 // SetTaskConfig sets a configuration value for a task type (legacy method - use typed setters above)
 // Note: SetTaskConfig was removed - use typed setters: SetVacuumTaskConfig, SetErasureCodingTaskConfig, SetBalanceTaskConfig, or SetReplicationTaskConfig
 
@@ -341,12 +207,11 @@ type MaintenanceQueue struct {
 
 // MaintenanceScanner analyzes the cluster and generates maintenance tasks
 type MaintenanceScanner struct {
-	adminClient      AdminClient
-	policy           *MaintenancePolicy
-	queue            *MaintenanceQueue
-	lastScan         map[MaintenanceTaskType]time.Time
-	integration      *MaintenanceIntegration
-	lastTopologyInfo *master_pb.TopologyInfo
+	adminClient AdminClient
+	policy      *MaintenancePolicy
+	queue       *MaintenanceQueue
+	lastScan    map[MaintenanceTaskType]time.Time
+	integration *MaintenanceIntegration
 }
 
 // TaskDetectionResult represents the result of scanning for maintenance needs
@@ -475,180 +340,6 @@ type ClusterReplicationTask struct {
 	Metadata        map[string]string `json:"metadata,omitempty"`
 }
 
-// BuildMaintenancePolicyFromTasks creates a maintenance policy with configurations
-// from all registered tasks using their UI providers
-func BuildMaintenancePolicyFromTasks() *MaintenancePolicy {
-	policy := &MaintenancePolicy{
-		TaskPolicies:                 make(map[string]*TaskPolicy),
-		GlobalMaxConcurrent:          4,
-		DefaultRepeatIntervalSeconds: 6 * 3600,  // 6 hours in seconds
-		DefaultCheckIntervalSeconds:  12 * 3600, // 12 hours in seconds
-	}
-
-	// Get all registered task types from the UI registry
-	uiRegistry := tasks.GetGlobalUIRegistry()
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-
-	for taskType, provider := range uiRegistry.GetAllProviders() {
-		// Convert task type to maintenance task type
-		maintenanceTaskType := MaintenanceTaskType(string(taskType))
-
-		// Get the default configuration from the UI provider
-		defaultConfig := provider.GetCurrentConfig()
-
-		// Create task policy from UI configuration
-		taskPolicy := &TaskPolicy{
-			Enabled:               true, // Default enabled
-			MaxConcurrent:         2,    // Default concurrency
-			RepeatIntervalSeconds: policy.DefaultRepeatIntervalSeconds,
-			CheckIntervalSeconds:  policy.DefaultCheckIntervalSeconds,
-		}
-
-		// Extract configuration using TaskConfig interface - no more map conversions!
-		if taskConfig, ok := defaultConfig.(interface{ ToTaskPolicy() *worker_pb.TaskPolicy }); ok {
-			// Use protobuf directly for clean, type-safe config extraction
-			pbTaskPolicy := taskConfig.ToTaskPolicy()
-			taskPolicy.Enabled = pbTaskPolicy.Enabled
-			taskPolicy.MaxConcurrent = pbTaskPolicy.MaxConcurrent
-			if pbTaskPolicy.RepeatIntervalSeconds > 0 {
-				taskPolicy.RepeatIntervalSeconds = pbTaskPolicy.RepeatIntervalSeconds
-			}
-			if pbTaskPolicy.CheckIntervalSeconds > 0 {
-				taskPolicy.CheckIntervalSeconds = pbTaskPolicy.CheckIntervalSeconds
-			}
-		}
-
-		// Also get defaults from scheduler if available (using types.TaskScheduler explicitly)
-		var scheduler types.TaskScheduler = typesRegistry.GetScheduler(taskType)
-		if scheduler != nil {
-			if taskPolicy.MaxConcurrent <= 0 {
-				taskPolicy.MaxConcurrent = int32(scheduler.GetMaxConcurrent())
-			}
-			// Convert default repeat interval to seconds
-			if repeatInterval := scheduler.GetDefaultRepeatInterval(); repeatInterval > 0 {
-				taskPolicy.RepeatIntervalSeconds = int32(repeatInterval.Seconds())
-			}
-		}
-
-		// Also get defaults from detector if available (using types.TaskDetector explicitly)
-		var detector types.TaskDetector = typesRegistry.GetDetector(taskType)
-		if detector != nil {
-			// Convert scan interval to check interval (seconds)
-			if scanInterval := detector.ScanInterval(); scanInterval > 0 {
-				taskPolicy.CheckIntervalSeconds = int32(scanInterval.Seconds())
-			}
-		}
-
-		policy.TaskPolicies[string(maintenanceTaskType)] = taskPolicy
-		glog.V(3).Infof("Built policy for task type %s: enabled=%v, max_concurrent=%d",
-			maintenanceTaskType, taskPolicy.Enabled, taskPolicy.MaxConcurrent)
-	}
-
-	glog.V(2).Infof("Built maintenance policy with %d task configurations", len(policy.TaskPolicies))
-	return policy
-}
-
-// SetPolicyFromTasks sets the maintenance policy from registered tasks
-func SetPolicyFromTasks(policy *MaintenancePolicy) {
-	if policy == nil {
-		return
-	}
-
-	// Build new policy from tasks
-	newPolicy := BuildMaintenancePolicyFromTasks()
-
-	// Copy task policies
-	policy.TaskPolicies = newPolicy.TaskPolicies
-
-	glog.V(1).Infof("Updated maintenance policy with %d task configurations from registered tasks", len(policy.TaskPolicies))
-}
-
-// GetTaskIcon returns the icon CSS class for a task type from its UI provider
-func GetTaskIcon(taskType MaintenanceTaskType) string {
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-	uiRegistry := tasks.GetGlobalUIRegistry()
-
-	// Convert MaintenanceTaskType to TaskType
-	for workerTaskType := range typesRegistry.GetAllDetectors() {
-		if string(workerTaskType) == string(taskType) {
-			// Get the UI provider for this task type
-			provider := uiRegistry.GetProvider(workerTaskType)
-			if provider != nil {
-				return provider.GetIcon()
-			}
-			break
-		}
-	}
-
-	// Default icon if no UI provider found
-	return "fas fa-cog text-muted"
-}
-
-// GetTaskDisplayName returns the display name for a task type from its UI provider
-func GetTaskDisplayName(taskType MaintenanceTaskType) string {
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-	uiRegistry := tasks.GetGlobalUIRegistry()
-
-	// Convert MaintenanceTaskType to TaskType
-	for workerTaskType := range typesRegistry.GetAllDetectors() {
-		if string(workerTaskType) == string(taskType) {
-			// Get the UI provider for this task type
-			provider := uiRegistry.GetProvider(workerTaskType)
-			if provider != nil {
-				return provider.GetDisplayName()
-			}
-			break
-		}
-	}
-
-	// Fallback to the task type string
-	return string(taskType)
-}
-
-// GetTaskDescription returns the description for a task type from its UI provider
-func GetTaskDescription(taskType MaintenanceTaskType) string {
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-	uiRegistry := tasks.GetGlobalUIRegistry()
-
-	// Convert MaintenanceTaskType to TaskType
-	for workerTaskType := range typesRegistry.GetAllDetectors() {
-		if string(workerTaskType) == string(taskType) {
-			// Get the UI provider for this task type
-			provider := uiRegistry.GetProvider(workerTaskType)
-			if provider != nil {
-				return provider.GetDescription()
-			}
-			break
-		}
-	}
-
-	// Fallback to a generic description
-	return "Configure detailed settings for " + string(taskType) + " tasks."
-}
-
-// BuildMaintenanceMenuItems creates menu items for all registered task types
-func BuildMaintenanceMenuItems() []*MaintenanceMenuItem {
-	var menuItems []*MaintenanceMenuItem
-
-	// Get all registered task types
-	registeredTypes := GetRegisteredMaintenanceTaskTypes()
-
-	for _, taskType := range registeredTypes {
-		menuItem := &MaintenanceMenuItem{
-			TaskType:    taskType,
-			DisplayName: GetTaskDisplayName(taskType),
-			Description: GetTaskDescription(taskType),
-			Icon:        GetTaskIcon(taskType),
-			IsEnabled:   IsMaintenanceTaskTypeRegistered(taskType),
-			Path:        "/maintenance/config/" + string(taskType),
-		}
-
-		menuItems = append(menuItems, menuItem)
-	}
-
-	return menuItems
-}
-
 // Helper functions to extract configuration fields
 
 // Note: Removed getVacuumConfigField, getErasureCodingConfigField, getBalanceConfigField, getReplicationConfigField

weed/admin/maintenance/maintenance_worker.go

@@ -1,421 +0,0 @@
-package maintenance
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"sync"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/worker"
-	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
-	"github.com/seaweedfs/seaweedfs/weed/worker/types"
-
-	// Import task packages to trigger their auto-registration
-	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
-	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
-	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
-)
-
-// MaintenanceWorkerService manages maintenance task execution
-// TaskExecutor defines the function signature for task execution
-type TaskExecutor func(*MaintenanceWorkerService, *MaintenanceTask) error
-
-// TaskExecutorFactory creates a task executor for a given worker service
-type TaskExecutorFactory func() TaskExecutor
-
-// Global registry for task executor factories
-var taskExecutorFactories = make(map[MaintenanceTaskType]TaskExecutorFactory)
-var executorRegistryMutex sync.RWMutex
-var executorRegistryInitOnce sync.Once
-
-// initializeExecutorFactories dynamically registers executor factories for all auto-registered task types
-func initializeExecutorFactories() {
-	executorRegistryInitOnce.Do(func() {
-		// Get all registered task types from the global registry
-		typesRegistry := tasks.GetGlobalTypesRegistry()
-
-		var taskTypes []MaintenanceTaskType
-		for workerTaskType := range typesRegistry.GetAllDetectors() {
-			// Convert types.TaskType to MaintenanceTaskType by string conversion
-			maintenanceTaskType := MaintenanceTaskType(string(workerTaskType))
-			taskTypes = append(taskTypes, maintenanceTaskType)
-		}
-
-		// Register generic executor for all task types
-		for _, taskType := range taskTypes {
-			RegisterTaskExecutorFactory(taskType, createGenericTaskExecutor)
-		}
-
-		glog.V(1).Infof("Dynamically registered generic task executor for %d task types: %v", len(taskTypes), taskTypes)
-	})
-}
-
-// RegisterTaskExecutorFactory registers a factory function for creating task executors
-func RegisterTaskExecutorFactory(taskType MaintenanceTaskType, factory TaskExecutorFactory) {
-	executorRegistryMutex.Lock()
-	defer executorRegistryMutex.Unlock()
-	taskExecutorFactories[taskType] = factory
-	glog.V(2).Infof("Registered executor factory for task type: %s", taskType)
-}
-
-// GetTaskExecutorFactory returns the factory for a task type
-func GetTaskExecutorFactory(taskType MaintenanceTaskType) (TaskExecutorFactory, bool) {
-	// Ensure executor factories are initialized
-	initializeExecutorFactories()
-
-	executorRegistryMutex.RLock()
-	defer executorRegistryMutex.RUnlock()
-	factory, exists := taskExecutorFactories[taskType]
-	return factory, exists
-}
-
-// GetSupportedExecutorTaskTypes returns all task types with registered executor factories
-func GetSupportedExecutorTaskTypes() []MaintenanceTaskType {
-	// Ensure executor factories are initialized
-	initializeExecutorFactories()
-
-	executorRegistryMutex.RLock()
-	defer executorRegistryMutex.RUnlock()
-
-	taskTypes := make([]MaintenanceTaskType, 0, len(taskExecutorFactories))
-	for taskType := range taskExecutorFactories {
-		taskTypes = append(taskTypes, taskType)
-	}
-	return taskTypes
-}
-
-// createGenericTaskExecutor creates a generic task executor that uses the task registry
-func createGenericTaskExecutor() TaskExecutor {
-	return func(mws *MaintenanceWorkerService, task *MaintenanceTask) error {
-		return mws.executeGenericTask(task)
-	}
-}
-
-// init does minimal initialization - actual registration happens lazily
-func init() {
-	// Executor factory registration will happen lazily when first accessed
-	glog.V(1).Infof("Maintenance worker initialized - executor factories will be registered on first access")
-}
-
-type MaintenanceWorkerService struct {
-	workerID      string
-	address       string
-	adminServer   string
-	capabilities  []MaintenanceTaskType
-	maxConcurrent int
-	currentTasks  map[string]*MaintenanceTask
-	queue         *MaintenanceQueue
-	adminClient   AdminClient
-	running       bool
-	stopChan      chan struct{}
-
-	// Task execution registry
-	taskExecutors map[MaintenanceTaskType]TaskExecutor
-
-	// Task registry for creating task instances
-	taskRegistry *tasks.TaskRegistry
-}
-
-// NewMaintenanceWorkerService creates a new maintenance worker service
-func NewMaintenanceWorkerService(workerID, address, adminServer string) *MaintenanceWorkerService {
-	// Get all registered maintenance task types dynamically
-	capabilities := GetRegisteredMaintenanceTaskTypes()
-
-	worker := &MaintenanceWorkerService{
-		workerID:      workerID,
-		address:       address,
-		adminServer:   adminServer,
-		capabilities:  capabilities,
-		maxConcurrent: 2, // Default concurrent task limit
-		currentTasks:  make(map[string]*MaintenanceTask),
-		stopChan:      make(chan struct{}),
-		taskExecutors: make(map[MaintenanceTaskType]TaskExecutor),
-		taskRegistry:  tasks.GetGlobalTaskRegistry(), // Use global registry with auto-registered tasks
-	}
-
-	// Initialize task executor registry
-	worker.initializeTaskExecutors()
-
-	glog.V(1).Infof("Created maintenance worker with %d registered task types", len(worker.taskRegistry.GetAll()))
-
-	return worker
-}
-
-// executeGenericTask executes a task using the task registry instead of hardcoded methods
-func (mws *MaintenanceWorkerService) executeGenericTask(task *MaintenanceTask) error {
-	glog.V(2).Infof("Executing generic task %s: %s for volume %d", task.ID, task.Type, task.VolumeID)
-
-	// Validate that task has proper typed parameters
-	if task.TypedParams == nil {
-		return fmt.Errorf("task %s has no typed parameters - task was not properly planned (insufficient destinations)", task.ID)
-	}
-
-	// Convert MaintenanceTask to types.TaskType
-	taskType := types.TaskType(string(task.Type))
-
-	// Create task instance using the registry
-	taskInstance, err := mws.taskRegistry.Get(taskType).Create(task.TypedParams)
-	if err != nil {
-		return fmt.Errorf("failed to create task instance: %w", err)
-	}
-
-	// Update progress to show task has started
-	mws.updateTaskProgress(task.ID, 5)
-
-	// Execute the task
-	err = taskInstance.Execute(context.Background(), task.TypedParams)
-	if err != nil {
-		return fmt.Errorf("task execution failed: %w", err)
-	}
-
-	// Update progress to show completion
-	mws.updateTaskProgress(task.ID, 100)
-
-	glog.V(2).Infof("Generic task %s completed successfully", task.ID)
-	return nil
-}
-
-// initializeTaskExecutors sets up the task execution registry dynamically
-func (mws *MaintenanceWorkerService) initializeTaskExecutors() {
-	mws.taskExecutors = make(map[MaintenanceTaskType]TaskExecutor)
-
-	// Get all registered executor factories and create executors
-	executorRegistryMutex.RLock()
-	defer executorRegistryMutex.RUnlock()
-
-	for taskType, factory := range taskExecutorFactories {
-		executor := factory()
-		mws.taskExecutors[taskType] = executor
-		glog.V(3).Infof("Initialized executor for task type: %s", taskType)
-	}
-
-	glog.V(2).Infof("Initialized %d task executors", len(mws.taskExecutors))
-}
-
-// RegisterTaskExecutor allows dynamic registration of new task executors
-func (mws *MaintenanceWorkerService) RegisterTaskExecutor(taskType MaintenanceTaskType, executor TaskExecutor) {
-	if mws.taskExecutors == nil {
-		mws.taskExecutors = make(map[MaintenanceTaskType]TaskExecutor)
-	}
-	mws.taskExecutors[taskType] = executor
-	glog.V(1).Infof("Registered executor for task type: %s", taskType)
-}
-
-// GetSupportedTaskTypes returns all task types that this worker can execute
-func (mws *MaintenanceWorkerService) GetSupportedTaskTypes() []MaintenanceTaskType {
-	return GetSupportedExecutorTaskTypes()
-}
-
-// Start begins the worker service
-func (mws *MaintenanceWorkerService) Start() error {
-	mws.running = true
-
-	// Register with admin server
-	worker := &MaintenanceWorker{
-		ID:            mws.workerID,
-		Address:       mws.address,
-		Capabilities:  mws.capabilities,
-		MaxConcurrent: mws.maxConcurrent,
-	}
-
-	if mws.queue != nil {
-		mws.queue.RegisterWorker(worker)
-	}
-
-	// Start worker loop
-	go mws.workerLoop()
-
-	glog.Infof("Maintenance worker %s started at %s", mws.workerID, mws.address)
-	return nil
-}
-
-// Stop terminates the worker service
-func (mws *MaintenanceWorkerService) Stop() {
-	mws.running = false
-	close(mws.stopChan)
-
-	// Wait for current tasks to complete or timeout
-	timeout := time.NewTimer(30 * time.Second)
-	defer timeout.Stop()
-
-	for len(mws.currentTasks) > 0 {
-		select {
-		case <-timeout.C:
-			glog.Warningf("Worker %s stopping with %d tasks still running", mws.workerID, len(mws.currentTasks))
-			return
-		case <-time.After(time.Second):
-			// Check again
-		}
-	}
-
-	glog.Infof("Maintenance worker %s stopped", mws.workerID)
-}
-
-// workerLoop is the main worker event loop
-func (mws *MaintenanceWorkerService) workerLoop() {
-	heartbeatTicker := time.NewTicker(30 * time.Second)
-	defer heartbeatTicker.Stop()
-
-	taskRequestTicker := time.NewTicker(5 * time.Second)
-	defer taskRequestTicker.Stop()
-
-	for mws.running {
-		select {
-		case <-mws.stopChan:
-			return
-		case <-heartbeatTicker.C:
-			mws.sendHeartbeat()
-		case <-taskRequestTicker.C:
-			mws.requestTasks()
-		}
-	}
-}
-
-// sendHeartbeat sends heartbeat to admin server
-func (mws *MaintenanceWorkerService) sendHeartbeat() {
-	if mws.queue != nil {
-		mws.queue.UpdateWorkerHeartbeat(mws.workerID)
-	}
-}
-
-// requestTasks requests new tasks from the admin server
-func (mws *MaintenanceWorkerService) requestTasks() {
-	if len(mws.currentTasks) >= mws.maxConcurrent {
-		return // Already at capacity
-	}
-
-	if mws.queue != nil {
-		task := mws.queue.GetNextTask(mws.workerID, mws.capabilities)
-		if task != nil {
-			mws.executeTask(task)
-		}
-	}
-}
-
-// executeTask executes a maintenance task
-func (mws *MaintenanceWorkerService) executeTask(task *MaintenanceTask) {
-	mws.currentTasks[task.ID] = task
-
-	go func() {
-		defer func() {
-			delete(mws.currentTasks, task.ID)
-		}()
-
-		glog.Infof("Worker %s executing task %s: %s", mws.workerID, task.ID, task.Type)
-
-		// Execute task using dynamic executor registry
-		var err error
-		if executor, exists := mws.taskExecutors[task.Type]; exists {
-			err = executor(mws, task)
-		} else {
-			err = fmt.Errorf("unsupported task type: %s", task.Type)
-			glog.Errorf("No executor registered for task type: %s", task.Type)
-		}
-
-		// Report task completion
-		if mws.queue != nil {
-			errorMsg := ""
-			if err != nil {
-				errorMsg = err.Error()
-			}
-			mws.queue.CompleteTask(task.ID, errorMsg)
-		}
-
-		if err != nil {
-			glog.Errorf("Worker %s failed to execute task %s: %v", mws.workerID, task.ID, err)
-		} else {
-			glog.Infof("Worker %s completed task %s successfully", mws.workerID, task.ID)
-		}
-	}()
-}
-
-// updateTaskProgress updates the progress of a task
-func (mws *MaintenanceWorkerService) updateTaskProgress(taskID string, progress float64) {
-	if mws.queue != nil {
-		mws.queue.UpdateTaskProgress(taskID, progress)
-	}
-}
-
-// GetStatus returns the current status of the worker
-func (mws *MaintenanceWorkerService) GetStatus() map[string]interface{} {
-	return map[string]interface{}{
-		"worker_id":      mws.workerID,
-		"address":        mws.address,
-		"running":        mws.running,
-		"capabilities":   mws.capabilities,
-		"max_concurrent": mws.maxConcurrent,
-		"current_tasks":  len(mws.currentTasks),
-		"task_details":   mws.currentTasks,
-	}
-}
-
-// SetQueue sets the maintenance queue for the worker
-func (mws *MaintenanceWorkerService) SetQueue(queue *MaintenanceQueue) {
-	mws.queue = queue
-}
-
-// SetAdminClient sets the admin client for the worker
-func (mws *MaintenanceWorkerService) SetAdminClient(client AdminClient) {
-	mws.adminClient = client
-}
-
-// SetCapabilities sets the worker capabilities
-func (mws *MaintenanceWorkerService) SetCapabilities(capabilities []MaintenanceTaskType) {
-	mws.capabilities = capabilities
-}
-
-// SetMaxConcurrent sets the maximum concurrent tasks
-func (mws *MaintenanceWorkerService) SetMaxConcurrent(max int) {
-	mws.maxConcurrent = max
-}
-
-// SetHeartbeatInterval sets the heartbeat interval (placeholder for future use)
-func (mws *MaintenanceWorkerService) SetHeartbeatInterval(interval time.Duration) {
-	// Future implementation for configurable heartbeat
-}
-
-// SetTaskRequestInterval sets the task request interval (placeholder for future use)
-func (mws *MaintenanceWorkerService) SetTaskRequestInterval(interval time.Duration) {
-	// Future implementation for configurable task requests
-}
-
-// MaintenanceWorkerCommand represents a standalone maintenance worker command
-type MaintenanceWorkerCommand struct {
-	workerService *MaintenanceWorkerService
-}
-
-// NewMaintenanceWorkerCommand creates a new worker command
-func NewMaintenanceWorkerCommand(workerID, address, adminServer string) *MaintenanceWorkerCommand {
-	return &MaintenanceWorkerCommand{
-		workerService: NewMaintenanceWorkerService(workerID, address, adminServer),
-	}
-}
-
-// Run starts the maintenance worker as a standalone service
-func (mwc *MaintenanceWorkerCommand) Run() error {
-	// Generate or load persistent worker ID if not provided
-	if mwc.workerService.workerID == "" {
-		// Get current working directory for worker ID persistence
-		wd, err := os.Getwd()
-		if err != nil {
-			return fmt.Errorf("failed to get working directory: %w", err)
-		}
-
-		workerID, err := worker.GenerateOrLoadWorkerID(wd)
-		if err != nil {
-			return fmt.Errorf("failed to generate or load worker ID: %w", err)
-		}
-		mwc.workerService.workerID = workerID
-	}
-
-	// Start the worker service
-	err := mwc.workerService.Start()
-	if err != nil {
-		return fmt.Errorf("failed to start maintenance worker: %w", err)
-	}
-
-	// Wait for interrupt signal
-	select {}
-}

weed/admin/plugin/plugin.go

@@ -122,6 +122,7 @@ type Plugin struct {
 type streamSession struct {
 	workerID  string
 	outgoing  chan *plugin_pb.AdminToWorkerMessage
+	done      chan struct{}
 	closeOnce sync.Once
 }
 
@@ -274,6 +275,7 @@ func (r *Plugin) WorkerStream(stream plugin_pb.PluginControlService_WorkerStream
 	session := &streamSession{
 		workerID: workerID,
 		outgoing: make(chan *plugin_pb.AdminToWorkerMessage, r.outgoingBuffer),
+		done:     make(chan struct{}),
 	}
 	r.putSession(session)
 	defer r.cleanupSession(workerID)
@@ -908,8 +910,10 @@ func (r *Plugin) sendLoop(
 			return nil
 		case <-r.shutdownCh:
 			return nil
-		case msg, ok := <-session.outgoing:
-			if !ok {
+		case <-session.done:
+			return nil
+		case msg := <-session.outgoing:
+			if msg == nil {
 				return nil
 			}
 			if err := stream.Send(msg); err != nil {
@@ -930,6 +934,8 @@ func (r *Plugin) sendToWorker(workerID string, message *plugin_pb.AdminToWorkerM
 	select {
 	case <-r.shutdownCh:
 		return fmt.Errorf("plugin is shutting down")
+	case <-session.done:
+		return fmt.Errorf("worker %s session is closed", workerID)
 	case session.outgoing <- message:
 		return nil
 	case <-time.After(r.sendTimeout):
@@ -1425,7 +1431,7 @@ func CloneConfigValueMap(in map[string]*plugin_pb.ConfigValue) map[string]*plugi
 
 func (s *streamSession) close() {
 	s.closeOnce.Do(func() {
-		close(s.outgoing)
+		close(s.done)
 	})
 }
 

weed/admin/plugin/plugin_cancel_test.go

@@ -26,7 +26,7 @@ func TestRunDetectionSendsCancelOnContextDone(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4)}
+	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4), done: make(chan struct{})}
 	pluginSvc.putSession(session)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -77,7 +77,7 @@ func TestExecuteJobSendsCancelOnContextDone(t *testing.T) {
 			{JobType: jobType, CanExecute: true, MaxExecutionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4)}
+	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4), done: make(chan struct{})}
 	pluginSvc.putSession(session)
 
 	job := &plugin_pb.JobSpec{JobId: "job-1", JobType: jobType}
@@ -135,8 +135,8 @@ func TestAdminScriptExecutionBlocksOtherDetection(t *testing.T) {
 			{JobType: "vacuum", CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
-	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
+	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
+	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
 	pluginSvc.putSession(adminSession)
 	pluginSvc.putSession(otherSession)
 
@@ -214,8 +214,8 @@ func TestAdminScriptExecutionBlocksOtherExecution(t *testing.T) {
 			{JobType: "vacuum", CanExecute: true, MaxExecutionConcurrency: 1},
 		},
 	})
-	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
-	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
+	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
+	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
 	pluginSvc.putSession(adminSession)
 	pluginSvc.putSession(otherSession)
 

weed/admin/plugin/plugin_detection_test.go

@@ -22,7 +22,7 @@ func TestRunDetectionIncludesLatestSuccessfulRun(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
+	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
 	pluginSvc.putSession(session)
 
 	oldSuccess := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
@@ -80,7 +80,7 @@ func TestRunDetectionOmitsLastSuccessfulRunWhenNoSuccessHistory(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
+	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
 	pluginSvc.putSession(session)
 
 	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
@@ -130,7 +130,7 @@ func TestRunDetectionWithReportCapturesDetectionActivities(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
+	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
 	pluginSvc.putSession(session)
 
 	reportCh := make(chan *DetectionReport, 1)
@@ -210,7 +210,7 @@ func TestRunDetectionAdminScriptUsesLastCompletedRun(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: "worker-admin-script", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
+	session := &streamSession{workerID: "worker-admin-script", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
 	pluginSvc.putSession(session)
 
 	successCompleted := time.Date(2026, 2, 1, 10, 0, 0, 0, time.UTC)

weed/admin/plugin/plugin_scheduler.go

@@ -95,16 +95,6 @@ func (r *Plugin) laneSchedulerLoop(ls *schedulerLaneState) {
 	}
 }
 
-// schedulerLoop is kept for backward compatibility; it delegates to
-// laneSchedulerLoop with the default lane. New code should not call this.
-func (r *Plugin) schedulerLoop() {
-	ls := r.lanes[LaneDefault]
-	if ls == nil {
-		ls = newLaneState(LaneDefault)
-	}
-	r.laneSchedulerLoop(ls)
-}
-
 // runLaneSchedulerIteration runs one scheduling pass for a single lane,
 // processing only the job types assigned to that lane.
 //
@@ -229,82 +219,6 @@ func (r *Plugin) runLaneSchedulerIterationConcurrent(ls *schedulerLaneState, job
 	return hadJobs.Load()
 }
 
-// runSchedulerIteration is kept for backward compatibility. It runs a
-// single iteration across ALL job types (equivalent to the old single-loop
-// behavior). It is only used by the legacy schedulerLoop() fallback.
-func (r *Plugin) runSchedulerIteration() bool {
-	ls := r.lanes[LaneDefault]
-	if ls == nil {
-		ls = newLaneState(LaneDefault)
-	}
-	// For backward compat, the old function processes all job types.
-	r.expireStaleJobs(time.Now().UTC())
-
-	jobTypes := r.registry.DetectableJobTypes()
-	if len(jobTypes) == 0 {
-		r.setSchedulerLoopState("", "idle")
-		return false
-	}
-
-	r.setSchedulerLoopState("", "waiting_for_lock")
-	releaseLock, err := r.acquireAdminLock("plugin scheduler iteration")
-	if err != nil {
-		glog.Warningf("Plugin scheduler failed to acquire lock: %v", err)
-		r.setSchedulerLoopState("", "idle")
-		return false
-	}
-	if releaseLock != nil {
-		defer releaseLock()
-	}
-
-	active := make(map[string]struct{}, len(jobTypes))
-	hadJobs := false
-
-	for _, jobType := range jobTypes {
-		active[jobType] = struct{}{}
-
-		policy, enabled, err := r.loadSchedulerPolicy(jobType)
-		if err != nil {
-			glog.Warningf("Plugin scheduler failed to load policy for %s: %v", jobType, err)
-			continue
-		}
-		if !enabled {
-			r.clearSchedulerJobType(jobType)
-			continue
-		}
-		initialDelay := time.Duration(0)
-		if runInfo := r.snapshotSchedulerRun(jobType); runInfo.lastRunStartedAt.IsZero() {
-			initialDelay = 5 * time.Second
-		}
-		if !r.markDetectionDue(jobType, policy.DetectionInterval, initialDelay) {
-			continue
-		}
-
-		detected := r.runJobTypeIteration(jobType, policy)
-		if detected {
-			hadJobs = true
-		}
-	}
-
-	r.pruneSchedulerState(active)
-	r.pruneDetectorLeases(active)
-	r.setSchedulerLoopState("", "idle")
-	return hadJobs
-}
-
-// wakeLane wakes the scheduler goroutine for a specific lane.
-func (r *Plugin) wakeLane(lane SchedulerLane) {
-	if r == nil {
-		return
-	}
-	if ls, ok := r.lanes[lane]; ok {
-		select {
-		case ls.wakeCh <- struct{}{}:
-		default:
-		}
-	}
-}
-
 // wakeAllLanes wakes all lane scheduler goroutines.
 func (r *Plugin) wakeAllLanes() {
 	if r == nil {

weed/admin/plugin/scheduler_status.go

@@ -210,16 +210,6 @@ func (r *Plugin) setSchedulerLoopStateForJobType(jobType, phase string) {
 	}
 }
 
-func (r *Plugin) recordSchedulerIterationComplete(hadJobs bool) {
-	if r == nil {
-		return
-	}
-	r.schedulerLoopMu.Lock()
-	r.schedulerLoopState.lastIterationHadJobs = hadJobs
-	r.schedulerLoopState.lastIterationCompleted = time.Now().UTC()
-	r.schedulerLoopMu.Unlock()
-}
-
 func (r *Plugin) snapshotSchedulerLoopState() schedulerLoopState {
 	if r == nil {
 		return schedulerLoopState{}

weed/admin/static/js/admin.js

@@ -1267,7 +1267,7 @@ async function submitUploadFile() {
         });
 
         // Send request
-        xhr.open('POST', '/api/files/upload');
+        xhr.open('POST', basePath('/api/files/upload'));
         xhr.send(formData);
 
     } catch (error) {
@@ -1320,7 +1320,7 @@ function exportFileList() {
 // Download file
 function downloadFile(filePath) {
     // Create download link using admin API
-    const downloadUrl = `/api/files/download?path=${encodeURIComponent(filePath)}`;
+    const downloadUrl = basePath(`/api/files/download?path=${encodeURIComponent(filePath)}`);
     window.open(downloadUrl, '_blank');
 }
 
@@ -1786,7 +1786,7 @@ function createFileViewerContent(file, content) {
     if (file.mime.startsWith('image/')) {
         return `
             <div class="text-center">
-                <img src="/api/files/download?path=${encodeURIComponent(file.full_path)}" 
+                <img src="${basePath('/api/files/download?path=' + encodeURIComponent(file.full_path))}"
                      class="img-fluid" alt="${file.name}" style="max-height: 500px;">
             </div>
         `;
@@ -1804,7 +1804,7 @@ function createFileViewerContent(file, content) {
     } else if (file.mime === 'application/pdf') {
         return `
             <div class="text-center">
-                <embed src="/api/files/download?path=${encodeURIComponent(file.full_path)}" 
+                <embed src="${basePath('/api/files/download?path=' + encodeURIComponent(file.full_path))}"
                        type="application/pdf" width="100%" height="500px">
             </div>
         `;

weed/admin/topology/topology_management.go

@@ -87,7 +87,7 @@ func (at *ActiveTopology) UpdateTopology(topologyInfo *master_pb.TopologyInfo) e
 					}
 
 					diskKey := fmt.Sprintf("%s:%d", nodeInfo.Id, diskInfo.DiskId)
-					glog.V(2).Infof("UpdateTopology: adding disk key=%q nodeId=%q diskId=%d diskType=%q address=%q grpcPort=%d volumes=%d maxVolumes=%d",
+					glog.V(3).Infof("UpdateTopology: adding disk key=%q nodeId=%q diskId=%d diskType=%q address=%q grpcPort=%d volumes=%d maxVolumes=%d",
 						diskKey, nodeInfo.Id, diskInfo.DiskId, diskType, nodeInfo.Address, nodeInfo.GrpcPort, diskInfo.VolumeCount, diskInfo.MaxVolumeCount)
 					node.disks[diskInfo.DiskId] = disk
 					at.disks[diskKey] = disk

weed/admin/view/app/object_store_users.templ

@@ -125,6 +125,9 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
                                                 <div class="d-flex align-items-center">
                                                     <i class="fas fa-user me-2 text-muted"></i>
                                                     <strong>{user.Username}</strong>
+                                                    if user.IsStatic {
+                                                        <span class="badge bg-secondary ms-2" title="Loaded from config file (read-only)">static</span>
+                                                    }
                                                 </div>
                                             </td>
                                             <td>{user.Email}</td>
@@ -133,24 +136,28 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
                                             </td>
                                             <td>
                                                 <div class="btn-group btn-group-sm" role="group">
-                                                    <button type="button" class="btn btn-outline-info" 
+                                                    <button type="button" class="btn btn-outline-info"
                                                             data-action="show-user-details" data-username={ user.Username }>
                                                         <i class="fas fa-info-circle"></i>
                                                     </button>
-                                                    <button type="button" class="btn btn-outline-primary" 
-                                                            data-action="edit-user" data-username={ user.Username }>
-                                                        <i class="fas fa-edit"></i>
-                                                    </button>
-                                                    if user.Username != "anonymous" {
+                                                    if !user.IsStatic {
+                                                        <button type="button" class="btn btn-outline-primary"
+                                                                data-action="edit-user" data-username={ user.Username }>
+                                                            <i class="fas fa-edit"></i>
+                                                        </button>
+                                                    }
+                                                    if user.Username != "anonymous" && !user.IsStatic {
                                                         <button type="button" class="btn btn-outline-secondary"
                                                                 data-action="manage-access-keys" data-username={ user.Username }>
                                                             <i class="fas fa-key"></i>
                                                         </button>
                                                     }
-                                                    <button type="button" class="btn btn-outline-danger" 
-                                                            data-action="delete-user" data-username={ user.Username }>
-                                                        <i class="fas fa-trash"></i>
-                                                    </button>
+                                                    if !user.IsStatic {
+                                                        <button type="button" class="btn btn-outline-danger"
+                                                                data-action="delete-user" data-username={ user.Username }>
+                                                            <i class="fas fa-trash"></i>
+                                                        </button>
+                                                    }
                                                 </div>
                                             </td>
                                         </tr>

weed/admin/view/app/s3_buckets.templ

@@ -163,7 +163,7 @@ templ S3Buckets(data dash.S3BucketsData) {
                                     for _, bucket := range data.Buckets {
                                         <tr>
                                             <td>
-                                                <a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
+                                                <a href={dash.PUrl(ctx, fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
                                                    class="text-decoration-none">
                                                     <i class="fas fa-cube me-2"></i>
                                                     {bucket.Name}
@@ -236,7 +236,7 @@ templ S3Buckets(data dash.S3BucketsData) {
                                             </td>
                                             <td>
                                                 <div class="btn-group btn-group-sm" role="group">
-                                                    <a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
+                                                    <a href={dash.PUrl(ctx, fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
                                                        class="btn btn-outline-success btn-sm"
                                                        title="Browse Files">
                                                         <i class="fas fa-folder-open"></i>

weed/admin/view/app/s3_buckets_templ.go

@@ -171,9 +171,9 @@ func S3Buckets(data dash.S3BucketsData) templ.Component {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var5 templ.SafeURL
-			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
+			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 166, Col: 123}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 166, Col: 124}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 			if templ_7745c5c3_Err != nil {
@@ -439,9 +439,9 @@ func S3Buckets(data dash.S3BucketsData) templ.Component {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var19 templ.SafeURL
-			templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
+			templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 239, Col: 127}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 239, Col: 128}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
 			if templ_7745c5c3_Err != nil {

weed/admin/view/app/s3tables_buckets.templ

@@ -2,6 +2,7 @@ package app
 
 import (
 	"fmt"
+	"net/url"
 
 	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
@@ -152,7 +153,7 @@ templ S3TablesBuckets(data dash.S3TablesBucketsData) {
 												<div class="btn-group btn-group-sm" role="group">
 													{{ bucketName, parseErr := s3tables.ParseBucketNameFromARN(bucket.ARN) }}
 													if parseErr == nil {
-														<a class="btn btn-outline-primary btn-sm" href={ templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)) }>
+														<a class="btn btn-outline-primary btn-sm" href={ dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName))) }>
 															<i class="fas fa-folder-open"></i>
 														</a>
 													} else {

weed/admin/view/app/s3tables_buckets_templ.go

@@ -10,6 +10,7 @@ import templruntime "github.com/a-h/templ/runtime"
 
 import (
 	"fmt"
+	"net/url"
 
 	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
@@ -48,7 +49,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var2 templ.SafeURL
 			templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("http://localhost:%d/v1/config", data.IcebergPort)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 23, Col: 124}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 24, Col: 124}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
 			if templ_7745c5c3_Err != nil {
@@ -66,7 +67,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 		var templ_7745c5c3_Var3 string
 		templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 30, Col: 91}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 31, Col: 91}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
 		if templ_7745c5c3_Err != nil {
@@ -84,7 +85,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var4 string
 			templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 37, Col: 109}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 38, Col: 109}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
 			if templ_7745c5c3_Err != nil {
@@ -97,7 +98,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var5 string
 			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 39, Col: 107}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 40, Col: 107}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 			if templ_7745c5c3_Err != nil {
@@ -120,7 +121,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 		var templ_7745c5c3_Var6 string
 		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalBuckets))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 69, Col: 47}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 70, Col: 47}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 		if templ_7745c5c3_Err != nil {
@@ -133,7 +134,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 		var templ_7745c5c3_Var7 string
 		templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04"))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 88, Col: 54}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 89, Col: 54}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
 		if templ_7745c5c3_Err != nil {
@@ -147,7 +148,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var8 string
 			templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 108, Col: 47}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 109, Col: 47}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
 			if templ_7745c5c3_Err != nil {
@@ -171,7 +172,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var9 string
 			templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 146, Col: 28}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 147, Col: 28}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
 			if templ_7745c5c3_Err != nil {
@@ -184,7 +185,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var10 string
 			templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.OwnerAccountID)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 147, Col: 38}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 148, Col: 38}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
 			if templ_7745c5c3_Err != nil {
@@ -197,7 +198,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var11 string
 			templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 148, Col: 52}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 149, Col: 52}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
 			if templ_7745c5c3_Err != nil {
@@ -210,7 +211,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var12 string
 			templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 149, Col: 52}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 150, Col: 52}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
 			if templ_7745c5c3_Err != nil {
@@ -223,7 +224,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var13 string
 			templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04"))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 150, Col: 60}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 151, Col: 60}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
 			if templ_7745c5c3_Err != nil {
@@ -240,9 +241,9 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 					return templ_7745c5c3_Err
 				}
 				var templ_7745c5c3_Var14 templ.SafeURL
-				templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)))
+				templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName))))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 155, Col: 149}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 156, Col: 166}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
 				if templ_7745c5c3_Err != nil {
@@ -265,7 +266,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var15 string
 			templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 163, Col: 122}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 164, Col: 122}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
 			if templ_7745c5c3_Err != nil {
@@ -278,7 +279,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var16 string
 			templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 166, Col: 126}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 167, Col: 126}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
 			if templ_7745c5c3_Err != nil {
@@ -291,7 +292,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var17 string
 			templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 169, Col: 128}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 170, Col: 128}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 			if templ_7745c5c3_Err != nil {
@@ -304,7 +305,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var18 string
 			templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 169, Col: 161}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 170, Col: 161}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
 			if templ_7745c5c3_Err != nil {
@@ -342,7 +343,7 @@ CREATE SECRET (
 
 SELECT * FROM iceberg_scan('s3://my-table-bucket/my-namespace/my-table');`)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 219, Col: 74}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 220, Col: 74}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
 			if templ_7745c5c3_Err != nil {
@@ -365,7 +366,7 @@ catalog = load_catalog(
 
 namespaces = catalog.list_namespaces()`)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 235, Col: 39}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 236, Col: 39}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
 			if templ_7745c5c3_Err != nil {

weed/admin/view/app/s3tables_tables.templ

@@ -24,7 +24,7 @@ templ S3TablesTables(data dash.S3TablesTablesData) {
 	<div class="mb-3">
 		{{ bucketName, parseErr := s3tables.ParseBucketNameFromARN(data.BucketARN) }}
 		if parseErr == nil {
-			<a href={ templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)) } class="btn btn-sm btn-outline-secondary">
+			<a href={ dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName))) } class="btn btn-sm btn-outline-secondary">
 				<i class="fas fa-arrow-left me-1"></i>Back to Namespaces
 			</a>
 		} else {
@@ -126,7 +126,7 @@ templ S3TablesTables(data dash.S3TablesTablesData) {
 											<td>
 												<div class="btn-group btn-group-sm" role="group">
 													if parseErr == nil {
-														<a class="btn btn-outline-primary btn-sm" href={ templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables/%s", url.PathEscape(bucketName), url.PathEscape(data.Namespace), url.PathEscape(tableName))) } title="View Iceberg Details">
+														<a class="btn btn-outline-primary btn-sm" href={ dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables/%s", url.PathEscape(bucketName), url.PathEscape(data.Namespace), url.PathEscape(tableName))) } title="View Iceberg Details">
 															<i class="fas fa-eye"></i>
 														</a>
 													} else {

weed/admin/view/app/s3tables_tables_templ.go

@@ -48,9 +48,9 @@ func S3TablesTables(data dash.S3TablesTablesData) templ.Component {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var2 templ.SafeURL
-			templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)))
+			templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName))))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_tables.templ`, Line: 27, Col: 99}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_tables.templ`, Line: 27, Col: 116}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
 			if templ_7745c5c3_Err != nil {
@@ -308,9 +308,9 @@ func S3TablesTables(data dash.S3TablesTablesData) templ.Component {
 					return templ_7745c5c3_Err
 				}
 				var templ_7745c5c3_Var17 templ.SafeURL
-				templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables/%s", url.PathEscape(bucketName), url.PathEscape(data.Namespace), url.PathEscape(tableName))))
+				templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables/%s", url.PathEscape(bucketName), url.PathEscape(data.Namespace), url.PathEscape(tableName))))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_tables.templ`, Line: 129, Col: 237}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_tables.templ`, Line: 129, Col: 238}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 				if templ_7745c5c3_Err != nil {

weed/admin/view/app/template_helpers.go

@@ -6,20 +6,6 @@ import (
 	"strings"
 )
 
-// getStatusColor returns Bootstrap color class for status
-func getStatusColor(status string) string {
-	switch status {
-	case "active", "healthy":
-		return "success"
-	case "warning":
-		return "warning"
-	case "critical", "unreachable":
-		return "danger"
-	default:
-		return "secondary"
-	}
-}
-
 // formatBytes converts bytes to human readable format
 func formatBytes(bytes int64) string {
 	if bytes == 0 {

weed/cluster/cluster.go

@@ -95,18 +95,6 @@ func NewCluster() *Cluster {
 	}
 }
 
-func (cluster *Cluster) getGroupMembers(filerGroup FilerGroupName, nodeType string, createIfNotFound bool) *GroupMembers {
-	switch nodeType {
-	case FilerType:
-		return cluster.filerGroups.getGroupMembers(filerGroup, createIfNotFound)
-	case BrokerType:
-		return cluster.brokerGroups.getGroupMembers(filerGroup, createIfNotFound)
-	case S3Type:
-		return cluster.s3Groups.getGroupMembers(filerGroup, createIfNotFound)
-	}
-	return nil
-}
-
 func (cluster *Cluster) AddClusterNode(ns, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
 	filerGroup := FilerGroupName(ns)
 	switch nodeType {

weed/cluster/lock_client.go

@@ -43,7 +43,8 @@ type LiveLock struct {
 	lc                  *LockClient
 	owner               string
 	lockTTL             time.Duration
-	consecutiveFailures int // Track connection failures to trigger fallback
+	consecutiveFailures int   // Track connection failures to trigger fallback
+	generation          int64 // fencing token from the lock server
 }
 
 // NewShortLivedLock creates a lock with a 5-second duration
@@ -214,6 +215,9 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
 		glog.V(4).Infof("LOCK: DistributedLock response - key=%s err=%v", lock.key, err)
 		if err == nil && resp != nil {
 			lock.renewToken = resp.RenewToken
+			if resp.Generation > 0 {
+				atomic.StoreInt64(&lock.generation, resp.Generation)
+			}
 			lock.consecutiveFailures = 0 // Reset failure counter on success
 			glog.V(4).Infof("LOCK: Got renewToken for key=%s", lock.key)
 		} else {
@@ -264,6 +268,12 @@ func (lock *LiveLock) LockOwner() string {
 	return lock.owner
 }
 
+// Generation returns the fencing token for this lock.
+// It increments on each fresh acquisition and stays the same on renewal.
+func (lock *LiveLock) Generation() int64 {
+	return atomic.LoadInt64(&lock.generation)
+}
+
 // IsLocked returns true if this instance currently holds the lock
 func (lock *LiveLock) IsLocked() bool {
 	return atomic.LoadInt32(&lock.isLocked) == 1

weed/cluster/lock_manager/distributed_lock_manager.go

@@ -13,10 +13,17 @@ const LiveLockTTL = time.Second * 7
 
 var NoLockServerError = fmt.Errorf("no lock server found")
 
+// ReplicateFunc is called to replicate a lock operation to a backup server.
+// The caller (filer server) provides this to avoid a circular dependency.
+// seq is a per-lock monotonic sequence number for causal ordering — the backup
+// rejects mutations with seq <= its current seq for that key.
+type ReplicateFunc func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool)
+
 type DistributedLockManager struct {
-	lockManager *LockManager
-	LockRing    *LockRing
-	Host        pb.ServerAddress
+	lockManager   *LockManager
+	LockRing      *LockRing
+	Host          pb.ServerAddress
+	ReplicateFn   ReplicateFunc // set by filer server after creation
 }
 
 func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
@@ -27,35 +34,51 @@ func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
 	}
 }
 
-func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (lockOwner string, renewToken string, movedTo pb.ServerAddress, err error) {
-	movedTo, err = dlm.findLockOwningFiler(key)
-	if err != nil {
-		return
-	}
-	if movedTo != dlm.Host {
-		return
-	}
-	lockOwner, renewToken, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
-	return
-}
-
-func (dlm *DistributedLockManager) findLockOwningFiler(key string) (movedTo pb.ServerAddress, err error) {
-	servers := dlm.LockRing.GetSnapshot()
-	if servers == nil {
+func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (lockOwner string, renewToken string, generation int64, movedTo pb.ServerAddress, err error) {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
 		err = NoLockServerError
 		return
 	}
-
-	movedTo = hashKeyToServer(key, servers)
+	if primary != dlm.Host {
+		// If this is a renewal (non-empty token) and we still hold the lock locally,
+		// serve it here rather than redirecting. This handles the window between
+		// ring update and lock transfer completion — the old primary remains
+		// authoritative for locks it still holds.
+		if token != "" {
+			if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
+				var seq int64
+				lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
+				if err == nil && renewToken != "" {
+					dlm.replicateToBackup(key, expiredAtNs, renewToken, owner, generation, seq, false)
+				}
+				return
+			}
+		}
+		movedTo = primary
+		return
+	}
+	var seq int64
+	lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
+	if err == nil && renewToken != "" {
+		dlm.replicateToBackup(key, expiredAtNs, renewToken, owner, generation, seq, false)
+	}
 	return
 }
 
 func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, movedTo pb.ServerAddress, err error) {
-	movedTo, err = dlm.findLockOwningFiler(key)
-	if err != nil {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
+		err = NoLockServerError
 		return
 	}
-	if movedTo != dlm.Host {
+	if primary != dlm.Host {
+		// If we still hold this lock locally, serve it here
+		if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup {
+			owner = lock.Owner
+			return
+		}
+		movedTo = primary
 		servers := dlm.LockRing.GetSnapshot()
 		glog.V(0).Infof("lock %s not on current %s but on %s from %v", key, dlm.Host, movedTo, servers)
 		return
@@ -65,26 +88,60 @@ func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, move
 }
 
 func (dlm *DistributedLockManager) Unlock(key string, token string) (movedTo pb.ServerAddress, err error) {
-	servers := dlm.LockRing.GetSnapshot()
-	if servers == nil {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
 		err = NoLockServerError
 		return
 	}
-
-	server := hashKeyToServer(key, servers)
-	if server != dlm.Host {
-		movedTo = server
+	if primary != dlm.Host {
+		// If we still hold this lock locally, serve the unlock here
+		if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
+			var isUnlocked bool
+			var generation int64
+			var seq int64
+			isUnlocked, generation, seq, err = dlm.lockManager.Unlock(key, token)
+			if isUnlocked {
+				dlm.replicateToBackup(key, 0, "", "", generation, seq, true)
+			}
+			return
+		}
+		movedTo = primary
 		return
 	}
-	_, err = dlm.lockManager.Unlock(key, token)
+	var isUnlocked bool
+	var generation int64
+	var seq int64
+	isUnlocked, generation, seq, err = dlm.lockManager.Unlock(key, token)
+	if isUnlocked {
+		dlm.replicateToBackup(key, 0, "", "", generation, seq, true)
+	}
 	return
 }
 
-// InsertLock is used to insert a lock to a server unconditionally
-// It is used when a server is down and the lock is moved to another server
-func (dlm *DistributedLockManager) InsertLock(key string, expiredAtNs int64, token string, owner string) {
-	dlm.lockManager.InsertLock(key, expiredAtNs, token, owner)
+// InsertLock is used to insert a lock to a server unconditionally.
+// It is used when a server is down and the lock is moved to another server.
+// After inserting, it replicates to the backup for this key.
+func (dlm *DistributedLockManager) InsertLock(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	if dlm.lockManager.InsertLock(key, expiredAtNs, token, owner, generation, seq) {
+		dlm.replicateToBackup(key, expiredAtNs, token, owner, generation, seq, false)
+	}
 }
+
+// InsertBackupLock inserts a lock as a backup copy, rejecting stale seq
+func (dlm *DistributedLockManager) InsertBackupLock(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	dlm.lockManager.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+}
+
+// RemoveBackupLock removes a backup lock unconditionally
+func (dlm *DistributedLockManager) RemoveBackupLock(key string) {
+	dlm.lockManager.RemoveLock(key)
+}
+
+// RemoveBackupLockIfSeq removes a local copy only if the incoming mutation is not older.
+func (dlm *DistributedLockManager) RemoveBackupLockIfSeq(key string, generation int64, seq int64) {
+	dlm.lockManager.RemoveBackupLockIfSeq(key, generation, seq)
+}
+
 func (dlm *DistributedLockManager) SelectNotOwnedLocks(servers []pb.ServerAddress) (locks []*Lock) {
 	return dlm.lockManager.SelectLocks(func(key string) bool {
 		server := hashKeyToServer(key, servers)
@@ -96,9 +153,40 @@ func (dlm *DistributedLockManager) CalculateTargetServer(key string, servers []p
 }
 
 func (dlm *DistributedLockManager) IsLocal(key string) bool {
-	servers := dlm.LockRing.GetSnapshot()
-	if len(servers) <= 1 {
+	primary := dlm.LockRing.GetPrimary(key)
+	if primary == "" {
 		return true
 	}
-	return hashKeyToServer(key, servers) == dlm.Host
+	return primary == dlm.Host
+}
+
+// AllLocks returns all non-expired locks on this node
+func (dlm *DistributedLockManager) AllLocks() []*Lock {
+	return dlm.lockManager.AllLocks()
+}
+
+// PromoteLock promotes a backup lock to primary
+func (dlm *DistributedLockManager) PromoteLock(key string) bool {
+	return dlm.lockManager.PromoteLock(key)
+}
+
+// DemoteLock demotes a primary lock to backup
+func (dlm *DistributedLockManager) DemoteLock(key string) bool {
+	return dlm.lockManager.DemoteLock(key)
+}
+
+// GetLock returns a copy of a lock if it exists
+func (dlm *DistributedLockManager) GetLock(key string) (*Lock, bool) {
+	return dlm.lockManager.GetLock(key)
+}
+
+// replicateToBackup asynchronously replicates a lock operation to the backup server
+func (dlm *DistributedLockManager) replicateToBackup(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+	_, backup := dlm.LockRing.GetPrimaryAndBackup(key)
+	if backup == "" {
+		return // single-server deployment, no backup
+	}
+	if dlm.ReplicateFn != nil {
+		go dlm.ReplicateFn(backup, key, expiredAtNs, token, owner, generation, seq, isUnlock)
+	}
 }

weed/cluster/lock_manager/distributed_lock_manager_test.go

@@ -0,0 +1,710 @@
+package lock_manager
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testCluster simulates a cluster of filer nodes with DLMs.
+// It wires up ReplicateFn so that replication calls arrive at the
+// correct peer's DLM, enabling end-to-end backup testing without gRPC.
+type testCluster struct {
+	mu    sync.Mutex
+	nodes map[pb.ServerAddress]*DistributedLockManager
+}
+
+func newTestCluster(hosts ...pb.ServerAddress) *testCluster {
+	c := &testCluster{nodes: make(map[pb.ServerAddress]*DistributedLockManager)}
+	servers := make([]pb.ServerAddress, len(hosts))
+	copy(servers, hosts)
+
+	for _, host := range hosts {
+		dlm := NewDistributedLockManager(host)
+		dlm.LockRing.SetSnapshot(servers, 0)
+		c.nodes[host] = dlm
+	}
+
+	// Wire up replication: each node's ReplicateFn calls the backup's DLM directly
+	for _, dlm := range c.nodes {
+		d := dlm // capture
+		d.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+			c.mu.Lock()
+			target, ok := c.nodes[server]
+			c.mu.Unlock()
+			if !ok {
+				return // server is down
+			}
+			if isUnlock {
+				target.RemoveBackupLockIfSeq(key, generation, seq)
+			} else {
+				target.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+			}
+		}
+	}
+
+	return c
+}
+
+func (c *testCluster) removeNode(host pb.ServerAddress) {
+	c.mu.Lock()
+	delete(c.nodes, host)
+	c.mu.Unlock()
+
+	// Update all remaining nodes' rings
+	remaining := c.getServers()
+	for _, dlm := range c.getNodes() {
+		dlm.LockRing.SetSnapshot(remaining, 0)
+	}
+}
+
+func (c *testCluster) addNode(host pb.ServerAddress) {
+	c.mu.Lock()
+	dlm := NewDistributedLockManager(host)
+	c.nodes[host] = dlm
+	c.mu.Unlock()
+
+	// Wire up replication
+	dlm.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+		c.mu.Lock()
+		target, ok := c.nodes[server]
+		c.mu.Unlock()
+		if !ok {
+			return
+		}
+		if isUnlock {
+			target.RemoveBackupLockIfSeq(key, generation, seq)
+		} else {
+			target.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+		}
+	}
+
+	servers := c.getServers()
+	for _, n := range c.getNodes() {
+		n.LockRing.SetSnapshot(servers, 0)
+	}
+}
+
+func (c *testCluster) getNodes() map[pb.ServerAddress]*DistributedLockManager {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	cp := make(map[pb.ServerAddress]*DistributedLockManager, len(c.nodes))
+	for k, v := range c.nodes {
+		cp[k] = v
+	}
+	return cp
+}
+
+func (c *testCluster) getServers() []pb.ServerAddress {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	var servers []pb.ServerAddress
+	for s := range c.nodes {
+		servers = append(servers, s)
+	}
+	return servers
+}
+
+func (c *testCluster) get(host pb.ServerAddress) *DistributedLockManager {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.nodes[host]
+}
+
+// acquireLock tries to acquire a lock on the correct primary node.
+// It follows redirects (movedTo) like a real client would.
+func (c *testCluster) acquireLock(key, owner string, ttl time.Duration) (renewToken string, generation int64, primaryHost pb.ServerAddress, err error) {
+	// Try any node first (simulates client connecting to seed filer)
+	for _, dlm := range c.getNodes() {
+		expiry := time.Now().Add(ttl).UnixNano()
+		var movedTo pb.ServerAddress
+		var lockErr error
+		_, renewToken, generation, movedTo, lockErr = dlm.LockWithTimeout(key, expiry, "", owner)
+		if movedTo != "" && movedTo != dlm.Host {
+			// Follow redirect
+			target := c.get(movedTo)
+			if target == nil {
+				err = fmt.Errorf("primary %s is down", movedTo)
+				return
+			}
+			_, renewToken, generation, _, lockErr = target.LockWithTimeout(key, expiry, "", owner)
+			if lockErr != nil {
+				err = lockErr
+				return
+			}
+			primaryHost = movedTo
+			// Wait briefly for async replication to complete
+			time.Sleep(10 * time.Millisecond)
+			return
+		}
+		if lockErr != nil {
+			err = lockErr
+			return
+		}
+		primaryHost = dlm.Host
+		time.Sleep(10 * time.Millisecond)
+		return
+	}
+	err = fmt.Errorf("no nodes available")
+	return
+}
+
+// renewLock renews a lock on the primary node
+func (c *testCluster) renewLock(key, owner, token string, ttl time.Duration, primaryHost pb.ServerAddress) (newToken string, generation int64, err error) {
+	target := c.get(primaryHost)
+	if target == nil {
+		err = fmt.Errorf("primary %s is down", primaryHost)
+		return
+	}
+	expiry := time.Now().Add(ttl).UnixNano()
+	var movedTo pb.ServerAddress
+	var lockErr error
+	_, newToken, generation, movedTo, lockErr = target.LockWithTimeout(key, expiry, token, owner)
+	if movedTo != "" && movedTo != primaryHost {
+		target = c.get(movedTo)
+		if target == nil {
+			err = fmt.Errorf("new primary %s is down", movedTo)
+			return
+		}
+		// Pass the existing token so the redirected renewal can match
+		// if the lock was already transferred to the new primary.
+		_, newToken, generation, _, lockErr = target.LockWithTimeout(key, expiry, token, owner)
+	}
+	err = lockErr
+	time.Sleep(10 * time.Millisecond)
+	return
+}
+
+// --- Test Cases ---
+
+func TestDLM_PrimaryCrash_BackupPromotes(t *testing.T) {
+	// Scenario: Lock is acquired, primary crashes, backup should have the lock
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-primary-crash"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+	require.NotEmpty(t, renewToken)
+
+	// Find the backup for this key
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+	require.NotEmpty(t, backup, "should have a backup server")
+
+	// Verify backup has the lock
+	backupDlm := cluster.get(backup)
+	backupLock, found := backupDlm.GetLock(key)
+	require.True(t, found, "backup should have the lock")
+	assert.True(t, backupLock.IsBackup, "lock on backup should be marked as backup")
+	assert.Equal(t, renewToken, backupLock.Token, "backup should have the same token")
+
+	// Crash the primary
+	cluster.removeNode(primaryHost)
+
+	// Simulate topology change: promote backup locks
+	for _, dlm := range cluster.getNodes() {
+		locks := dlm.AllLocks()
+		for _, lock := range locks {
+			newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if newPrimary == dlm.Host && lock.IsBackup {
+				dlm.PromoteLock(lock.Key)
+			}
+		}
+	}
+
+	// The backup should now be the primary
+	newPrimary := backupDlm.LockRing.GetPrimary(key)
+	assert.Equal(t, backup, newPrimary, "backup should be the new primary")
+
+	// The promoted lock should work — verify it's no longer a backup
+	promotedLock, found := backupDlm.GetLock(key)
+	require.True(t, found, "lock should still exist after promotion")
+	assert.False(t, promotedLock.IsBackup, "lock should be promoted to primary")
+
+	// Client should be able to renew with the same token on the new primary
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, backup)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "renewal on new primary should succeed")
+}
+
+func TestDLM_BackupCrash_PrimaryContinues(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-backup-crash"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Crash the backup
+	cluster.removeNode(backup)
+
+	// Primary should still work — renew the lock
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "primary should continue working after backup crash")
+
+	// Verify primary is still the primary for this key
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	assert.Equal(t, primaryHost, newPrimary)
+}
+
+func TestDLM_BothPrimaryAndBackupCrash(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-both-crash"
+	_, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Crash both
+	cluster.removeNode(primaryHost)
+	cluster.removeNode(backup)
+
+	// The lock is lost — the surviving node should be able to acquire it fresh
+	newToken, _, _, err := cluster.acquireLock(key, "owner2", 30*time.Second)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "new owner should acquire lock after both crash")
+}
+
+func TestDLM_RollingRestart(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire multiple locks
+	type lockState struct {
+		key, owner, token string
+		generation        int64
+		primary           pb.ServerAddress
+	}
+	locks := make([]lockState, 5)
+	for i := range locks {
+		key := fmt.Sprintf("rolling-lock-%d", i)
+		token, gen, primary, err := cluster.acquireLock(key, fmt.Sprintf("owner-%d", i), 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockState{key: key, owner: fmt.Sprintf("owner-%d", i), token: token, generation: gen, primary: primary}
+	}
+
+	// Rolling restart: remove and re-add each node one at a time.
+	// After removing a node, promote backups and re-replicate to new backups
+	// to maintain the invariant that each lock has a backup copy.
+	for _, host := range hosts {
+		cluster.removeNode(host)
+
+		// Simulate full OnDlmChangeSnapshot: promote backups and re-replicate
+		for _, dlm := range cluster.getNodes() {
+			for _, lock := range dlm.AllLocks() {
+				newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+				if newPrimary == dlm.Host && lock.IsBackup {
+					dlm.PromoteLock(lock.Key)
+				}
+			}
+			// Re-replicate all primary locks to their new backups
+			for _, lock := range dlm.AllLocks() {
+				newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+				if newPrimary == dlm.Host && !lock.IsBackup {
+					dlm.replicateToBackup(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq, false)
+				}
+			}
+		}
+
+		time.Sleep(10 * time.Millisecond)
+
+		// Re-add the node
+		cluster.addNode(host)
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	// After rolling restart, locks should survive via backup promotion
+	survivedCount := 0
+	for _, ls := range locks {
+		for _, dlm := range cluster.getNodes() {
+			lock, found := dlm.GetLock(ls.key)
+			if found && !lock.IsBackup {
+				survivedCount++
+				break
+			}
+		}
+	}
+	t.Logf("Locks survived rolling restart: %d / %d", survivedCount, len(locks))
+	require.Greater(t, survivedCount, 0, "at least some locks should survive a rolling restart via backup promotion")
+}
+
+func TestDLM_GenerationIncrementsOnNewAcquisition(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "gen-test-lock"
+
+	// Acquire lock — generation should be > 0
+	token1, gen1, primary, err := cluster.acquireLock(key, "owner1", 2*time.Second)
+	require.NoError(t, err)
+	assert.Greater(t, gen1, int64(0))
+
+	// Renew — generation should stay the same
+	token2, gen2, err := cluster.renewLock(key, "owner1", token1, 2*time.Second, primary)
+	require.NoError(t, err)
+	assert.Equal(t, gen1, gen2, "generation should not change on renewal")
+
+	// Let lock expire
+	time.Sleep(3 * time.Second)
+
+	// Re-acquire — generation should increment
+	_, gen3, _, err := cluster.acquireLock(key, "owner2", 30*time.Second)
+	require.NoError(t, err)
+	assert.Greater(t, gen3, gen1, "generation should increment on new acquisition")
+	_ = token2
+}
+
+func TestDLM_ReplicationFailure_PrimaryStillWorks(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Break replication by setting a no-op ReplicateFn on all nodes
+	for _, dlm := range cluster.getNodes() {
+		dlm.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+			// Simulate replication failure: do nothing
+		}
+	}
+
+	key := "repl-fail-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	// Primary should have the lock
+	primaryDlm := cluster.get(primaryHost)
+	lock, found := primaryDlm.GetLock(key)
+	require.True(t, found, "primary should have the lock")
+	assert.False(t, lock.IsBackup)
+
+	// Backup should NOT have it (replication failed)
+	_, backup := primaryDlm.LockRing.GetPrimaryAndBackup(key)
+	backupDlm := cluster.get(backup)
+	_, found = backupDlm.GetLock(key)
+	assert.False(t, found, "backup should not have the lock when replication fails")
+
+	// Primary should still be able to renew
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken)
+}
+
+func TestDLM_UnlockReplicatesToBackup(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "unlock-repl-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Verify backup has the lock
+	_, found := cluster.get(backup).GetLock(key)
+	require.True(t, found, "backup should have the lock")
+
+	// Unlock on primary
+	primaryDlm := cluster.get(primaryHost)
+	movedTo, err := primaryDlm.Unlock(key, renewToken)
+	require.NoError(t, err)
+	assert.Empty(t, movedTo)
+
+	// Wait for async replication
+	time.Sleep(20 * time.Millisecond)
+
+	// Backup should also have removed the lock
+	_, found = cluster.get(backup).GetLock(key)
+	assert.False(t, found, "backup should remove lock after unlock replication")
+}
+
+func TestDLM_TopologyChange_LockSurvivesServerAddition(t *testing.T) {
+	// Start with 2 servers, acquire lock, add a 3rd server
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "topo-add-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	// Add a new server
+	cluster.addNode("filer3:8888")
+	time.Sleep(20 * time.Millisecond)
+
+	// The lock should still be accessible — either the same primary or on a new one
+	// Try to renew on the original primary first
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	if newPrimary == primaryHost {
+		// Still on same primary
+		newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+		require.NoError(t, err)
+		assert.NotEmpty(t, newToken)
+	}
+	// If primary changed, the lock may need transfer — that's handled by OnDlmChangeSnapshot
+	// which is tested at the server level
+}
+
+func TestDLM_ConsistentHashing_MinimalDisruption(t *testing.T) {
+	// Verify that removing a server only affects locks on that server
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire 50 locks
+	type lockInfo struct {
+		key, token string
+		primary    pb.ServerAddress
+	}
+	locks := make([]lockInfo, 50)
+	for i := range locks {
+		key := fmt.Sprintf("min-disrupt-%d", i)
+		token, _, primary, err := cluster.acquireLock(key, "owner", 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockInfo{key: key, token: token, primary: primary}
+	}
+
+	// Count locks per server before removal
+	countBefore := make(map[pb.ServerAddress]int)
+	for _, l := range locks {
+		countBefore[l.primary]++
+	}
+	t.Logf("Lock distribution before: %v", countBefore)
+
+	// Remove filer2
+	cluster.removeNode("filer2:8888")
+
+	// Count how many locks changed primary
+	changed := 0
+	for _, l := range locks {
+		// Check where the lock should be now
+		for _, dlm := range cluster.getNodes() {
+			newPrimary := dlm.LockRing.GetPrimary(l.key)
+			if newPrimary != l.primary {
+				changed++
+			}
+			break
+		}
+	}
+
+	// Only locks from filer2 should have changed
+	assert.Equal(t, countBefore["filer2:8888"], changed,
+		"only locks from removed server should change primary")
+}
+
+func TestDLM_NodeDropAndJoin_OwnershipDisruption(t *testing.T) {
+	// Scenario: 3 nodes, acquire locks, one drops and a NEW node joins quickly.
+	// The new node steals hash ranges from surviving nodes, not just from the
+	// departed node. This test measures the disruption.
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire many locks
+	numLocks := 100
+	type lockInfo struct {
+		key, token string
+		primary    pb.ServerAddress
+	}
+	locks := make([]lockInfo, numLocks)
+	for i := range locks {
+		key := fmt.Sprintf("churn-lock-%d", i)
+		token, _, primary, err := cluster.acquireLock(key, "owner", 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockInfo{key: key, token: token, primary: primary}
+	}
+
+	// Record primary for each lock before the change
+	beforePrimary := make(map[string]pb.ServerAddress)
+	for _, l := range locks {
+		beforePrimary[l.key] = l.primary
+	}
+
+	// Drop filer3 and immediately add filer4
+	cluster.removeNode("filer3:8888")
+
+	// Promote backups on remaining nodes (simulates OnDlmChangeSnapshot)
+	for _, dlm := range cluster.getNodes() {
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p == dlm.Host && lock.IsBackup {
+				dlm.PromoteLock(lock.Key)
+			}
+		}
+		// Re-replicate primary locks to new backups
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p == dlm.Host && !lock.IsBackup {
+				dlm.replicateToBackup(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq, false)
+			}
+		}
+	}
+	time.Sleep(10 * time.Millisecond)
+
+	// Now add filer4 (new node, empty)
+	cluster.addNode("filer4:8888")
+	time.Sleep(10 * time.Millisecond)
+
+	// Simulate OnDlmChangeSnapshot on all nodes after filer4 joins:
+	// transfer locks that now belong to filer4
+	for host, dlm := range cluster.getNodes() {
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p != host && !lock.IsBackup {
+				// This lock should move to the new primary
+				target := cluster.get(p)
+				if target != nil {
+					target.InsertLock(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq)
+					dlm.DemoteLock(lock.Key)
+				}
+			}
+		}
+	}
+	time.Sleep(10 * time.Millisecond)
+
+	// Count disruptions: locks whose primary changed to a node other than filer3's successor
+	disruptedFromSurvivors := 0
+	disruptedFromDeparted := 0
+	movedToFiler4 := 0
+	for _, l := range locks {
+		// What's the new primary?
+		var newPrimary pb.ServerAddress
+		for _, dlm := range cluster.getNodes() {
+			newPrimary = dlm.LockRing.GetPrimary(l.key)
+			break
+		}
+		oldPrimary := beforePrimary[l.key]
+		if newPrimary != oldPrimary {
+			if oldPrimary == "filer3:8888" {
+				disruptedFromDeparted++
+			} else {
+				disruptedFromSurvivors++
+			}
+		}
+		if newPrimary == "filer4:8888" {
+			movedToFiler4++
+		}
+	}
+
+	t.Logf("Locks disrupted from departed filer3: %d / %d", disruptedFromDeparted, numLocks)
+	t.Logf("Locks disrupted from surviving filer1/filer2: %d / %d", disruptedFromSurvivors, numLocks)
+	t.Logf("Locks now on new filer4: %d / %d", movedToFiler4, numLocks)
+
+	// The key concern: filer4 joining disrupts locks on surviving nodes
+	// With consistent hashing, new node steals ~1/N of each surviving node's keys
+	// Verify that the transfer logic above moved those locks to filer4
+	for _, l := range locks {
+		var newPrimary pb.ServerAddress
+		for _, dlm := range cluster.getNodes() {
+			newPrimary = dlm.LockRing.GetPrimary(l.key)
+			break
+		}
+		target := cluster.get(newPrimary)
+		require.NotNil(t, target, "primary %s should exist", newPrimary)
+
+		lock, found := target.GetLock(l.key)
+		if !found {
+			// Lock may have only a backup copy if transfer happened but
+			// the lock was on the departed node and wasn't re-replicated.
+			// Check all nodes for any copy.
+			anyFound := false
+			for _, dlm := range cluster.getNodes() {
+				if _, f := dlm.GetLock(l.key); f {
+					anyFound = true
+					break
+				}
+			}
+			if !anyFound {
+				t.Errorf("lock %s completely lost (primary should be %s)", l.key, newPrimary)
+			}
+			continue
+		}
+		assert.False(t, lock.IsBackup, "lock %s on primary %s should not be a backup", l.key, newPrimary)
+	}
+}
+
+func TestDLM_RenewalDuringTransferWindow(t *testing.T) {
+	// When a new node joins and steals a key range from a surviving node,
+	// there's a window between ring update and lock transfer. During this
+	// window, a client renewal should still succeed on the old primary
+	// (because it still holds the lock locally).
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Find a key that will move primary when filer4 is added.
+	// Try candidate keys until we find one whose primary changes.
+	var key, renewToken string
+	var primaryHost pb.ServerAddress
+	for i := 0; i < 1000; i++ {
+		candidate := fmt.Sprintf("transfer-window-lock-%d", i)
+		token, _, primary, err := cluster.acquireLock(candidate, "owner1", 30*time.Second)
+		require.NoError(t, err)
+
+		// Check if adding filer4 would move this key's primary
+		tmpRing := NewHashRing(DefaultVnodeCount)
+		tmpRing.SetServers([]pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888", "filer4:8888"})
+		newPrimary := tmpRing.GetPrimary(candidate)
+		if newPrimary != primary {
+			key = candidate
+			renewToken = token
+			primaryHost = primary
+			break
+		}
+	}
+	require.NotEmpty(t, key, "should find a key that moves primary when filer4 joins")
+
+	// Add filer4 — this changes the primary for our key per the ring
+	cluster.addNode("filer4:8888")
+	time.Sleep(10 * time.Millisecond)
+
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	require.NotEqual(t, primaryHost, newPrimary, "key should have moved to a different primary")
+
+	// Renewal on the OLD primary should still succeed because it holds the lock locally
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err, "renewal on old primary should succeed during transfer window")
+	assert.NotEmpty(t, newToken, "should get a new token from old primary")
+	t.Logf("Key %s: primary changed from %s to %s, but renewal on old primary succeeded", key, primaryHost, newPrimary)
+}
+
+func TestDLM_StaleReplicationRejected(t *testing.T) {
+	// Verify that a stale replication (lower seq) does not overwrite a newer one
+	lm := NewLockManager()
+
+	// Insert backup with seq=3
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 1, 3)
+	lock, found := lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(3), lock.Seq)
+
+	// Try to overwrite with stale seq=2 — should be rejected
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 1, 2)
+	lock, found = lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-new", lock.Token, "stale replication should be rejected")
+	assert.Equal(t, int64(3), lock.Seq)
+
+	// Update with higher seq=4 — should succeed
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-newer", "owner1", 1, 4)
+	lock, found = lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-newer", lock.Token, "newer replication should be accepted")
+	assert.Equal(t, int64(4), lock.Seq)
+
+	// Stale unlock (seq=2) should not delete the lock
+	removed := lm.RemoveBackupLockIfSeq("key1", 1, 2)
+	assert.False(t, removed, "stale unlock should be rejected")
+	_, found = lm.GetLock("key1")
+	assert.True(t, found, "lock should still exist after stale unlock")
+
+	// Valid unlock (seq=5) should delete
+	removed = lm.RemoveBackupLockIfSeq("key1", 1, 5)
+	assert.True(t, removed, "valid unlock should be accepted")
+	_, found = lm.GetLock("key1")
+	assert.False(t, found, "lock should be removed after valid unlock")
+}

weed/cluster/lock_manager/hash_ring.go

@@ -0,0 +1,198 @@
+package lock_manager
+
+import (
+	"hash/crc32"
+	"sort"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+const DefaultVnodeCount = 50
+
+// HashRing implements consistent hashing with virtual nodes.
+// When a server is removed, only the keys that hashed to that server
+// are remapped (to the next server on the ring), leaving all other
+// key-to-server mappings stable.
+//
+// UPGRADE NOTE: This replaces the previous modulo-based hashing
+// (hash % len(servers)). The two schemes compute different primaries
+// for the same key, so all filer nodes in the cluster must be upgraded
+// together (or via a rolling restart that completes within the lock TTL
+// window of ~7 seconds) to avoid routing disagreements.
+type HashRing struct {
+	mu            sync.RWMutex
+	vnodeCount    int
+	sortedHashes  []uint32                     // sorted ring positions
+	vnodeToServer map[uint32]pb.ServerAddress   // ring position → server
+	servers       map[pb.ServerAddress]struct{} // set of all servers
+}
+
+func NewHashRing(vnodeCount int) *HashRing {
+	if vnodeCount <= 0 {
+		vnodeCount = DefaultVnodeCount
+	}
+	return &HashRing{
+		vnodeCount:    vnodeCount,
+		vnodeToServer: make(map[uint32]pb.ServerAddress),
+		servers:       make(map[pb.ServerAddress]struct{}),
+	}
+}
+
+// AddServer adds a server with virtual nodes to the ring.
+func (hr *HashRing) AddServer(server pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	if _, exists := hr.servers[server]; exists {
+		return
+	}
+	hr.servers[server] = struct{}{}
+	hr.rebuildRing()
+}
+
+// RemoveServer removes a server and its virtual nodes from the ring.
+func (hr *HashRing) RemoveServer(server pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	if _, exists := hr.servers[server]; !exists {
+		return
+	}
+	delete(hr.servers, server)
+	hr.rebuildRing()
+}
+
+// SetServers replaces the entire server set.
+func (hr *HashRing) SetServers(servers []pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	hr.servers = make(map[pb.ServerAddress]struct{}, len(servers))
+	for _, s := range servers {
+		hr.servers[s] = struct{}{}
+	}
+	hr.rebuildRing()
+}
+
+// GetPrimaryAndBackup returns the primary server for a key and its backup
+// (the next distinct server clockwise on the ring).
+// If there is only one server, backup is empty.
+func (hr *HashRing) GetPrimaryAndBackup(key string) (primary, backup pb.ServerAddress) {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	if len(hr.sortedHashes) == 0 {
+		return "", ""
+	}
+
+	hash := hashKey(key)
+	idx := hr.search(hash)
+	primary = hr.vnodeToServer[hr.sortedHashes[idx]]
+
+	// Walk clockwise to find a different server for backup
+	ringLen := len(hr.sortedHashes)
+	for i := 1; i < ringLen; i++ {
+		candidate := hr.vnodeToServer[hr.sortedHashes[(idx+i)%ringLen]]
+		if candidate != primary {
+			backup = candidate
+			return
+		}
+	}
+	// Only one server — no backup
+	return primary, ""
+}
+
+// GetPrimary returns just the primary server for a key.
+func (hr *HashRing) GetPrimary(key string) pb.ServerAddress {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	if len(hr.sortedHashes) == 0 {
+		return ""
+	}
+
+	hash := hashKey(key)
+	idx := hr.search(hash)
+	return hr.vnodeToServer[hr.sortedHashes[idx]]
+}
+
+// GetServers returns a sorted copy of all servers in the ring.
+func (hr *HashRing) GetServers() []pb.ServerAddress {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	servers := make([]pb.ServerAddress, 0, len(hr.servers))
+	for s := range hr.servers {
+		servers = append(servers, s)
+	}
+	sort.Slice(servers, func(i, j int) bool {
+		return servers[i] < servers[j]
+	})
+	return servers
+}
+
+// ServerCount returns the number of servers in the ring.
+func (hr *HashRing) ServerCount() int {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+	return len(hr.servers)
+}
+
+// rebuildRing rebuilds the sorted hash ring from the current server set.
+// Caller must hold hr.mu write lock.
+func (hr *HashRing) rebuildRing() {
+	hr.vnodeToServer = make(map[uint32]pb.ServerAddress, len(hr.servers)*hr.vnodeCount)
+	hr.sortedHashes = make([]uint32, 0, len(hr.servers)*hr.vnodeCount)
+
+	for server := range hr.servers {
+		for i := 0; i < hr.vnodeCount; i++ {
+			vnodeKey := vnodeKeyFor(server, i)
+			hash := hashKey(vnodeKey)
+			hr.vnodeToServer[hash] = server
+			hr.sortedHashes = append(hr.sortedHashes, hash)
+		}
+	}
+	sort.Slice(hr.sortedHashes, func(i, j int) bool {
+		return hr.sortedHashes[i] < hr.sortedHashes[j]
+	})
+}
+
+// search finds the first ring position >= hash.
+func (hr *HashRing) search(hash uint32) int {
+	idx := sort.Search(len(hr.sortedHashes), func(i int) bool {
+		return hr.sortedHashes[i] >= hash
+	})
+	if idx >= len(hr.sortedHashes) {
+		idx = 0 // wrap around
+	}
+	return idx
+}
+
+func hashKey(key string) uint32 {
+	return crc32.ChecksumIEEE([]byte(key))
+}
+
+func vnodeKeyFor(server pb.ServerAddress, index int) string {
+	// Use a format that distributes well across the ring
+	buf := make([]byte, 0, len(server)+10)
+	buf = append(buf, []byte(server)...)
+	buf = append(buf, '#')
+	buf = appendInt(buf, index)
+	return string(buf)
+}
+
+func appendInt(buf []byte, n int) []byte {
+	if n == 0 {
+		return append(buf, '0')
+	}
+	// Simple int-to-string without importing strconv
+	digits := [20]byte{}
+	pos := len(digits)
+	for n > 0 {
+		pos--
+		digits[pos] = byte('0' + n%10)
+		n /= 10
+	}
+	return append(buf, digits[pos:]...)
+}