feat(admin): add profiling options for debugging high memory/CPU usage (#8923 )

* feat(admin): add profiling options for debugging high memory/CPU usage Add -debug, -debug.port, -cpuprofile, and -memprofile flags to the admin command, matching the profiling support already available in master, volume, and other server commands. This enables investigation of resource usage issues like #8919. * refactor(admin): move profiling flags into AdminOptions struct Move cpuprofile and memprofile flags from global variables into the AdminOptions struct and init() function for consistency with other flags. * fix(debug): bind pprof server to localhost only and document profiling flags StartDebugServer was binding to all interfaces (0.0.0.0), exposing runtime profiling data to the network. Restrict to 127.0.0.1 since this is a development/debugging tool. Also add a "Debugging and Profiling" section to the admin command's help text documenting the new flags.
fix(volume-rust): fix volume balance between Go and Rust servers (#8915 )
2026-10-07 07:06:33 +00:00 · 2026-04-04 10:05:19 -07:00 · 2026-04-04 09:13:23 -07:00 · 2026-04-03 20:01:28 -07:00 · 2026-04-03 19:57:30 -07:00 · 2026-04-03 18:37:54 -07:00
459 changed files with 13068 additions and 49147 deletions
@@ -0,0 +1 @@
+{"sessionId":"d6574c47-eafc-4a94-9dce-f9ffea22b53c","pid":10111,"acquiredAt":1775248373916}
@@ -23,6 +23,11 @@ on:
          - all
          - standard
          - large_disk
+      publish:
+        description: 'Publish images and manifests'
+        required: true
+        type: boolean
+        default: false

 permissions:
  contents: read
@@ -33,6 +38,7 @@ jobs:
    runs-on: ubuntu-latest
    outputs:
      variants: ${{ steps.set-variants.outputs.variants }}
+      publish: ${{ steps.set-publish.outputs.publish }}
    steps:
      - name: Select variants for this run
        id: set-variants
@@ -43,6 +49,14 @@ jobs:
            variants='["standard","large_disk"]'
          fi
          echo "variants=$variants" >> "$GITHUB_OUTPUT"
+      - name: Select publish mode
+        id: set-publish
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
+          else
+            echo "publish=true" >> "$GITHUB_OUTPUT"
+          fi

  build:
    needs: [setup]
@@ -112,13 +126,13 @@ jobs:
          buildkitd-flags: "--debug"
          buildkitd-config: /tmp/buildkitd.toml
      - name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
+        if: needs.setup.outputs.publish == 'true'
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Login to GHCR
-        if: github.event_name != 'pull_request'
+        if: needs.setup.outputs.publish == 'true'
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
@@ -130,7 +144,7 @@ jobs:
          DOCKER_BUILDKIT: 1
        with:
          context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ needs.setup.outputs.publish == 'true' }}
          file: ./docker/Dockerfile.go_build
          platforms: linux/${{ matrix.platform }}
          # Push to GHCR only during build to avoid Docker Hub rate limits
@@ -166,33 +180,112 @@ jobs:
            echo "tag_suffix=" >> $GITHUB_OUTPUT
          fi
      - name: Login to GHCR
+        if: needs.setup.outputs.publish == 'true'
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ secrets.GHCR_USERNAME }}
          password: ${{ secrets.GHCR_TOKEN }}
-      - name: Run Trivy vulnerability scanner
-        # Pin to SHA — mutable tags were compromised (GHSA-69fq-xp46-6x23)
+      - name: Checkout for local scan build
+        if: needs.setup.outputs.publish != 'true'
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
+      - name: Create BuildKit config for local scan build
+        if: needs.setup.outputs.publish != 'true'
+        run: |
+          cat > /tmp/buildkitd.toml <<EOF
+          [registry."docker.io"]
+            mirrors = ["https://mirror.gcr.io"]
+          EOF
+      - name: Set up Docker Buildx for local scan build
+        if: needs.setup.outputs.publish != 'true'
+        uses: docker/setup-buildx-action@v4
+        with:
+          buildkitd-flags: "--debug"
+          buildkitd-config: /tmp/buildkitd.toml
+      - name: Build local scan image tarball
+        if: needs.setup.outputs.publish != 'true'
+        uses: docker/build-push-action@v7
+        env:
+          DOCKER_BUILDKIT: 1
+        with:
+          context: ./docker
+          file: ./docker/Dockerfile.go_build
+          platforms: linux/amd64
+          outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1
+            BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
+            ${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
+      - name: Trivy report (published image)
+        if: needs.setup.outputs.publish == 'true'
+        # Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
        with:
-          # Scan amd64 only — OS packages are identical across architectures
+          scan-type: image
+          # Scan amd64 only - OS packages are identical across architectures
          # since they all use the same alpine base, so a single-arch scan
          # provides sufficient coverage without multiplying CI time.
          image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
+          scanners: vuln
+          vuln-type: os,library
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          limit-severities-for-sarif: true
          format: sarif
          output: trivy-results.sarif
+          exit-code: '0'
+      - name: Trivy report (local tarball)
+        if: needs.setup.outputs.publish != 'true'
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
+          scanners: vuln
+          vuln-type: os,library
          severity: HIGH,CRITICAL
-          exit-code: '1'
+          ignore-unfixed: true
+          limit-severities-for-sarif: true
+          format: sarif
+          output: trivy-results.sarif
+          exit-code: '0'
      - name: Upload Trivy scan results to GitHub Security
-        uses: github/codeql-action/upload-sarif@v3
+        uses: github/codeql-action/upload-sarif@v4
        if: always()
        with:
          sarif_file: trivy-results.sarif
+      - name: Trivy gate (published image)
+        if: needs.setup.outputs.publish == 'true'
+        # Gate only on fixable high/critical vulnerabilities. Non-fixable
+        # findings are still visible in the SARIF upload above.
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          scan-type: image
+          image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
+          scanners: vuln
+          vuln-type: os,library
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          format: table
+          exit-code: '1'
+          skip-setup-trivy: true
+      - name: Trivy gate (local tarball)
+        if: needs.setup.outputs.publish != 'true'
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
+          scanners: vuln
+          vuln-type: os,library
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          format: table
+          exit-code: '1'
+          skip-setup-trivy: true

  create-manifest:
    runs-on: ubuntu-latest
    needs: [setup, build, trivy-scan]
-    if: github.event_name != 'pull_request'
+    if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
    strategy:
      matrix:
        variant: ${{ fromJSON(needs.setup.outputs.variants) }}
@@ -39,7 +39,80 @@ concurrency:
  cancel-in-progress: false

 jobs:
+
+  # ── Pre-build Rust volume server binaries natively ──────────────────
+  # Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour
+  # emulated cargo build into ~15 minutes of native compilation.
+  build-rust-binaries:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - target: x86_64-unknown-linux-musl
+            arch: amd64
+          - target: aarch64-unknown-linux-musl
+            arch: arm64
+            cross: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install protobuf compiler
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
+
+      - name: Install musl tools (amd64)
+        if: ${{ !matrix.cross }}
+        run: sudo apt-get install -y musl-tools
+
+      - name: Install cross-compilation tools (arm64)
+        if: matrix.cross
+        run: |
+          sudo apt-get install -y gcc-aarch64-linux-gnu
+          echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
+
+      - name: Cache cargo registry and target
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            seaweed-volume/target
+          key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
+          restore-keys: |
+            rust-docker-${{ matrix.target }}-
+
+      - name: Build large-disk variant
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }}
+          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
+
+      - name: Build normal variant
+        env:
+          SEAWEEDFS_COMMIT: ${{ github.sha }}
+        run: |
+          cd seaweed-volume
+          cargo build --release --target ${{ matrix.target }} --no-default-features
+          cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: rust-volume-${{ matrix.arch }}
+          path: |
+            weed-volume-large-disk-${{ matrix.arch }}
+            weed-volume-normal-${{ matrix.arch }}
+
+  # ── Build Docker containers ─────────────────────────────────────────
  build:
+    needs: [build-rust-binaries]
    runs-on: ubuntu-latest
    strategy:
      # Build sequentially to avoid rate limits
@@ -52,20 +125,23 @@ jobs:
            dockerfile: ./docker/Dockerfile.go_build
            build_args: ""
            tag_suffix: ""
-            
-          # Large disk - multi-arch  
+            rust_variant: normal
+
+          # Large disk - multi-arch
          - variant: large_disk
            platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
            dockerfile: ./docker/Dockerfile.go_build
            build_args: TAGS=5BytesOffset
            tag_suffix: _large_disk
-            
+            rust_variant: large-disk
+
          # Full tags - multi-arch
          - variant: full
            platforms: linux/amd64,linux/arm64
            dockerfile: ./docker/Dockerfile.go_build
            build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
            tag_suffix: _full
+            rust_variant: normal

          # Large disk + full tags - multi-arch
          - variant: large_disk_full
@@ -73,19 +149,42 @@ jobs:
            dockerfile: ./docker/Dockerfile.go_build
            build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
            tag_suffix: _large_disk_full
-            
+            rust_variant: large-disk
+
          # RocksDB large disk - amd64 only
          - variant: rocksdb
            platforms: linux/amd64
            dockerfile: ./docker/Dockerfile.rocksdb_large
            build_args: ""
            tag_suffix: _large_disk_rocksdb
+            rust_variant: large-disk
            
    steps:
      - name: Checkout
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        uses: actions/checkout@v6
-        
+
+      - name: Download pre-built Rust binaries
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+        uses: actions/download-artifact@v4
+        with:
+          pattern: rust-volume-*
+          merge-multiple: true
+          path: ./rust-bins
+
+      - name: Place Rust binaries in Docker context
+        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
+        run: |
+          mkdir -p docker/weed-volume-prebuilt
+          for arch in amd64 arm64; do
+            src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
+            if [ -f "$src" ]; then
+              cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
+              echo "Placed pre-built Rust binary for ${arch}"
+            fi
+          done
+          ls -la docker/weed-volume-prebuilt/
+
      - name: Free Disk Space
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
        run: |
@@ -21,7 +21,7 @@ jobs:
          fetch-depth: 0

      - name: Set up Helm
-        uses: azure/setup-helm@v4
+        uses: azure/setup-helm@v5
        with:
          version: v3.18.4

@@ -43,7 +43,7 @@ jobs:
        uses: dtolnay/rust-toolchain@stable

      - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
        with:
          path: |
            ~/.cargo/registry
@@ -80,7 +80,7 @@ jobs:
        uses: dtolnay/rust-toolchain@stable

      - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
        with:
          path: |
            ~/.cargo/registry
@@ -93,7 +93,7 @@ jobs:
      - name: Build Go weed binary
        run: |
          cd weed
-          go build -o weed .
+          go build -tags 5BytesOffset -o weed .
          chmod +x weed
          ./weed version

@@ -156,7 +156,7 @@ jobs:
        uses: dtolnay/rust-toolchain@stable

      - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
        with:
          path: |
            ~/.cargo/registry
@@ -169,7 +169,7 @@ jobs:
      - name: Build Go weed binary
        run: |
          cd weed
-          go build -o weed .
+          go build -tags 5BytesOffset -o weed .
          chmod +x weed
          ./weed version

@@ -47,7 +47,7 @@ jobs:
        uses: dtolnay/rust-toolchain@stable

      - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
        with:
          path: |
            ~/.cargo/registry
@@ -118,7 +118,7 @@ jobs:
          targets: ${{ matrix.target }}

      - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
        with:
          path: |
            ~/.cargo/registry
@@ -39,11 +39,19 @@ jobs:
      - name: Install cross-compilation tools
        if: matrix.cross
        run: |
-          sudo apt-get install -y gcc-aarch64-linux-gnu
+          sudo dpkg --add-architecture arm64
+          sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
+          echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
+          echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
+          sudo apt-get update
+          sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
          echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
+          echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"
+          echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"
+          echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"

      - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
        with:
          path: |
            ~/.cargo/registry
@@ -80,6 +88,7 @@ jobs:
          rm weed-volume-normal

      - name: Upload release assets
+        if: startsWith(github.ref, 'refs/tags/')
        uses: softprops/action-gh-release@v2
        with:
          files: |
@@ -88,6 +97,15 @@ jobs:
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

+      - name: Upload artifacts
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: rust-volume-${{ matrix.asset_suffix }}
+          path: |
+            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
+            weed-volume_${{ matrix.asset_suffix }}.tar.gz
+
  build-rust-volume-darwin:
    permissions:
      contents: write
@@ -112,7 +130,7 @@ jobs:
          targets: ${{ matrix.target }}

      - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
        with:
          path: |
            ~/.cargo/registry
@@ -147,6 +165,7 @@ jobs:
          rm weed-volume-normal

      - name: Upload release assets
+        if: startsWith(github.ref, 'refs/tags/')
        uses: softprops/action-gh-release@v2
        with:
          files: |
@@ -155,6 +174,15 @@ jobs:
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

+      - name: Upload artifacts
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: rust-volume-${{ matrix.asset_suffix }}
+          path: |
+            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
+            weed-volume_${{ matrix.asset_suffix }}.tar.gz
+
  build-rust-volume-windows:
    permissions:
      contents: write
@@ -170,7 +198,7 @@ jobs:
        uses: dtolnay/rust-toolchain@stable

      - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
        with:
          path: |
            ~/.cargo/registry
@@ -206,6 +234,7 @@ jobs:
          rm weed-volume-normal.exe

      - name: Upload release assets
+        if: startsWith(github.ref, 'refs/tags/')
        uses: softprops/action-gh-release@v2
        with:
          files: |
@@ -213,3 +242,12 @@ jobs:
            weed-volume_windows_amd64.zip
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload artifacts
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: rust-volume-windows_amd64
+          path: |
+            weed-volume_large_disk_windows_amd64.zip
+            weed-volume_windows_amd64.zip
@@ -0,0 +1,5 @@
+{
+  "setup": [],
+  "teardown": [],
+  "run": []
+}
@@ -16,15 +16,22 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
  && CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"

-# Rust volume server builder (amd64/arm64 only)
-FROM rust:1-alpine as rust_builder
+# Rust volume server: use pre-built binary from CI when available (placed in
+# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
+# from source.  Pre-building avoids a multi-hour QEMU-emulated cargo build
+# for non-native architectures.
+FROM alpine:3.23 as rust_builder
 ARG TARGETARCH
-RUN apk add musl-dev protobuf-dev git
-COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
-COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/proto /build/proto
-WORKDIR /build/seaweed-volume
 ARG TAGS
-RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
+COPY weed-volume-prebuilt/ /prebuilt/
+COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
+COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
+WORKDIR /build/seaweed-volume
+RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
+      echo "Using pre-built Rust binary for ${TARGETARCH}" && \
+      cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \
+    elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
+      apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
      if [ "$TAGS" = "5BytesOffset" ]; then \
        cargo build --release; \
      else \
@@ -49,7 +56,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
 # To disable: docker run -e GODEBUG=fips140=off ...

 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse curl su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse curl su-exec libgcc && \
    addgroup -g 1000 seaweed && \
    adduser -D -u 1000 -G seaweed seaweed

@@ -79,9 +87,5 @@ RUN mkdir -p /data/filerldb2 && \
 VOLUME /data
 WORKDIR /data

-# Run as non-root by default (satisfies security scanners).
-# Use `docker run --user root` if you need the entrypoint to fix
-# /data volume ownership before dropping privileges.
-USER seaweed
-
+# Entrypoint will handle permission fixes and user switching
 ENTRYPOINT ["/entrypoint.sh"]
@@ -7,7 +7,8 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
 COPY ./entrypoint.sh /entrypoint.sh

 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse curl su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse curl su-exec && \
    addgroup -g 1000 seaweed && \
    adduser -D -u 1000 -G seaweed seaweed

@@ -37,9 +38,5 @@ RUN mkdir -p /data/filerldb2 && \
 VOLUME /data
 WORKDIR /data

-# Run as non-root by default (satisfies security scanners).
-# Use `docker run --user root` if you need the entrypoint to fix
-# /data volume ownership before dropping privileges.
-USER seaweed
-
+# Entrypoint will handle permission fixes and user switching
 ENTRYPOINT ["/entrypoint.sh"]
@@ -34,7 +34,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh

 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse snappy gflags curl su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse snappy gflags curl su-exec && \
    addgroup -g 1000 seaweed && \
    adduser -D -u 1000 -G seaweed seaweed

@@ -17,7 +17,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh

 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse snappy gflags curl tmux su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse snappy gflags curl tmux su-exec && \
    addgroup -g 1000 seaweed && \
    adduser -D -u 1000 -G seaweed seaweed

@@ -0,0 +1,53 @@
+{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        {
+          "accessKey": "AKIAIOSFODNN7EXAMPLE",
+          "secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    },
+    {
+      "name": "steward",
+      "credentials": [
+        {
+          "accessKey": "steward-key",
+          "secretKey": "steward-secret"
+        }
+      ],
+      "actions": [
+        "Read",
+        "List",
+        "Write"
+      ]
+    },
+    {
+      "name": "le001",
+      "credentials": [
+        {
+          "accessKey": "le001-key",
+          "secretKey": "le001-secret"
+        }
+      ],
+      "actions": [
+        "Read",
+        "List"
+      ]
+    },
+    {
+      "name": "anonymous",
+      "actions": [
+        "Read"
+      ]
+    }
+  ]
+}
@@ -83,10 +83,10 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/viant/ptrie v1.0.1
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/scram v1.2.0
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	go.etcd.io/etcd/client/v3 v3.6.7
+	go.etcd.io/etcd/client/v3 v3.6.9
 	go.mongodb.org/mongo-driver v1.17.9
 	go.opencensus.io v0.24.0 // indirect
 	gocloud.dev v0.45.0
@@ -94,7 +94,7 @@ require (
 	gocloud.dev/pubsub/rabbitpubsub v0.45.0
 	golang.org/x/crypto v0.49.0
 	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
-	golang.org/x/image v0.36.0
+	golang.org/x/image v0.38.0
 	golang.org/x/net v0.51.0
 	golang.org/x/oauth2 v0.36.0
 	golang.org/x/sys v0.42.0
@@ -114,7 +114,7 @@ require (
 )

 require (
-	cloud.google.com/go/kms v1.25.0
+	cloud.google.com/go/kms v1.26.0
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
 	github.com/Jille/raft-grpc-transport v1.6.1
 	github.com/ThreeDotsLabs/watermill v1.5.1
@@ -124,13 +124,13 @@ require (
 	github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
 	github.com/arangodb/go-driver v1.6.9
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.41.4
-	github.com/aws/aws-sdk-go-v2/config v1.32.9
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.12
+	github.com/aws/aws-sdk-go-v2 v1.41.5
+	github.com/aws/aws-sdk-go-v2/config v1.32.13
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.13
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
-	github.com/getsentry/sentry-go v0.43.0
+	github.com/getsentry/sentry-go v0.44.1
 	github.com/go-ldap/ldap/v3 v3.4.13
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
@@ -157,7 +157,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
 	github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
-	go.etcd.io/etcd/client/pkg/v3 v3.6.7
+	go.etcd.io/etcd/client/pkg/v3 v3.6.9
 	go.uber.org/atomic v1.11.0
 	golang.org/x/sync v0.20.0
 	golang.org/x/tools/godoc v0.1.0-deprecated
@@ -181,7 +181,7 @@ require (
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
 	github.com/apache/thrift v0.22.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/bazelbuild/rules_go v0.46.0 // indirect
 	github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
@@ -324,21 +324,21 @@ require (
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.9
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
 	github.com/aws/smithy-go v1.24.2
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
@@ -371,7 +371,7 @@ require (
 	github.com/geoffgarside/ber v1.2.0 // indirect
 	github.com/go-chi/chi/v5 v5.2.5 // indirect
 	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
@@ -486,7 +486,7 @@ require (
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
 	go.etcd.io/bbolt v1.4.3 // indirect
-	go.etcd.io/etcd/api/v3 v3.6.7 // indirect
+	go.etcd.io/etcd/api/v3 v3.6.9 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
 cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
 cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
 cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
-cloud.google.com/go/kms v1.25.0 h1:gVqvGGUmz0nYCmtoxWmdc1wli2L1apgP8U4fghPGSbQ=
-cloud.google.com/go/kms v1.25.0/go.mod h1:XIdHkzfj0bUO3E+LvwPg+oc7s58/Ns8Nd8Sdtljihbk=
+cloud.google.com/go/kms v1.26.0 h1:cK9mN2cf+9V63D3H1f6koxTatWy39aTI/hCjz1I+adU=
+cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58=
 cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
 cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
 cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
@@ -714,48 +714,48 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
-github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
+github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
+github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A=
-github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
+github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
+github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
 github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
 github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
 github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
 github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
@@ -1051,8 +1051,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
 github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
 github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
 github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
-github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
-github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
+github.com/getsentry/sentry-go v0.44.1 h1:/cPtrA5qB7uMRrhgSn9TYtcEF36auGP3Y6+ThvD/yaI=
+github.com/getsentry/sentry-go v0.44.1/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
 github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
@@ -1076,8 +1076,8 @@ github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
-github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
+github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -1838,8 +1838,6 @@ github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1Ivohy
 github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
 github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564 h1:TgxPraf1NmF6XTcUG53ULpLQrKvhtUJxQ3hyekxSDNQ=
 github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
-github.com/seaweedfs/go-fuse/v2 v2.9.1 h1:gnKmfrKreCRGJmekGz5WMnNZqXEf9s9+V2hdWQdvx88=
-github.com/seaweedfs/go-fuse/v2 v2.9.1/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
 github.com/seaweedfs/go-fuse/v2 v2.9.2 h1:IfP/yFjLGO4rALcJY2Gb39PlebHxLnj7dkIiQAjFres=
 github.com/seaweedfs/go-fuse/v2 v2.9.2/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
 github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
@@ -2038,8 +2036,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
-github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
+github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -2103,12 +2101,12 @@ go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
 go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
 go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
-go.etcd.io/etcd/api/v3 v3.6.7 h1:7BNJ2gQmc3DNM+9cRkv7KkGQDayElg8x3X+tFDYS+E0=
-go.etcd.io/etcd/api/v3 v3.6.7/go.mod h1:xJ81TLj9hxrYYEDmXTeKURMeY3qEDN24hqe+q7KhbnI=
-go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/AhBVs=
-go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
-go.etcd.io/etcd/client/v3 v3.6.7 h1:9WqA5RpIBtdMxAy1ukXLAdtg2pAxNqW5NUoO2wQrE6U=
-go.etcd.io/etcd/client/v3 v3.6.7/go.mod h1:2XfROY56AXnUqGsvl+6k29wrwsSbEh1lAouQB1vHpeE=
+go.etcd.io/etcd/api/v3 v3.6.9 h1:UA7iKfEW1AzgihcBSGXci2kDGQiokSq41F9HMCI/RTI=
+go.etcd.io/etcd/api/v3 v3.6.9/go.mod h1:csEk/qTfxKL36NqJdU15Tgtl65A8dyEY2BYo7PRsIwk=
+go.etcd.io/etcd/client/pkg/v3 v3.6.9 h1:T8nuk8Lz64C+Hzb0coBFLMSlVSQZBpAtFk46swdM1DA=
+go.etcd.io/etcd/client/pkg/v3 v3.6.9/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
+go.etcd.io/etcd/client/v3 v3.6.9 h1:3X555hQXmhRr27O37wls53g68CpUiPOiHXrZfz2Al+o=
+go.etcd.io/etcd/client/v3 v3.6.9/go.mod h1:KO7H1HLYh1qaljuVZJQwBFk1lRce6pJzt+C81GEnrlM=
 go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
 go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -2242,8 +2240,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
 golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
-golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
-golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
+golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
+golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.17"
+appVersion: "4.18"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.17.0
+version: 4.18.0
@@ -84,6 +84,8 @@ service SeaweedFiler {
    // distributed lock management internal use only
    rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
    }
+    rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
+    }
 }

 //////////////////////////////////////////////////
@@ -522,6 +524,7 @@ message LockResponse {
    string lock_owner = 2;
    string lock_host_moved_to = 3;
    string error = 4;
+    int64 generation = 5;
 }
 message UnlockRequest {
    string name = 1;
@@ -544,12 +547,26 @@ message Lock {
    string renew_token = 2;
    int64 expired_at_ns = 3;
    string owner = 4;
+    int64 generation = 5;
+    bool is_backup = 6;
+    int64 seq = 7;
 }
 message TransferLocksRequest {
    repeated Lock locks = 1;
 }
 message TransferLocksResponse {
 }
+message ReplicateLockRequest {
+    string name = 1;
+    string renew_token = 2;
+    int64 expired_at_ns = 3;
+    string owner = 4;
+    int64 generation = 5;
+    bool is_unlock = 6;
+    int64 seq = 7;
+}
+message ReplicateLockResponse {
+}

 //////////////////////////////////////////////////
 // StreamMutateEntry: ordered bidirectional streaming for all filer mutations.
@@ -2561,6 +2561,15 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"

+[[package]]
+name = "openssl-src"
+version = "300.5.5+3.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f1787d533e03597a7934fd0a765f0d28e94ecc5fb7789f8053b1e699a56f709"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "openssl-sys"
 version = "0.9.111"
@@ -2569,6 +2578,7 @@ checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321"
 dependencies = [
 "cc",
 "libc",
+ "openssl-src",
 "pkg-config",
 "vcpkg",
 ]
@@ -4654,6 +4664,7 @@ dependencies = [
 "memmap2",
 "mime_guess",
 "multer",
+ "openssl",
 "parking_lot 0.12.5",
 "pprof",
 "prometheus",
@@ -20,7 +20,7 @@ default = ["5bytes"]
 [dependencies]
 # Async runtime
 tokio = { version = "1", features = ["full"] }
-tokio-stream = "0.1"
+tokio-stream = { version = "0.1", features = ["net"] }
 tokio-io-timeout = "1"

 # gRPC + protobuf
@@ -65,8 +65,6 @@ reed-solomon-erasure = "6"
 # Logging
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-pprof = { version = "0.15", features = ["prost-codec"] }
-
 # Config
 toml = "0.8"
 serde = { version = "1", features = ["derive"] }
@@ -127,6 +125,10 @@ aws-sdk-s3 = { version = "1.125.0", default-features = false, features = ["sigv4
 aws-credential-types = "1"
 aws-types = "1"

+# pprof is Unix-only (requires libc/nix APIs not available on Windows)
+[target.'cfg(unix)'.dependencies]
+pprof = { version = "0.15", features = ["prost-codec"] }
+
 [dev-dependencies]
 tempfile = "3"

@@ -10,9 +10,11 @@ use seaweed_volume::security::tls::{
    GrpcClientAuthPolicy, TlsPolicy,
 };
 use seaweed_volume::security::{Guard, SigningKey};
+#[cfg(unix)]
 use seaweed_volume::server::debug::build_debug_router;
 use seaweed_volume::server::grpc_client::load_outgoing_grpc_tls;
 use seaweed_volume::server::grpc_server::VolumeGrpcService;
+#[cfg(unix)]
 use seaweed_volume::server::profiling::CpuProfileSession;
 use seaweed_volume::server::request_id::GrpcRequestIdLayer;
 use seaweed_volume::server::volume_server::{
@@ -24,6 +26,11 @@ use seaweed_volume::storage::types::DiskType;

 use tokio_rustls::TlsAcceptor;

+#[cfg(unix)]
+type CpuProfileParam = Option<CpuProfileSession>;
+#[cfg(not(unix))]
+type CpuProfileParam = Option<()>;
+
 const GRPC_MAX_MESSAGE_SIZE: usize = 1 << 30;
 const GRPC_KEEPALIVE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);
 const GRPC_KEEPALIVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
@@ -42,6 +49,7 @@ fn main() {

    let config = config::parse_cli();
    seaweed_volume::server::server_stats::init_process_start();
+    #[cfg(unix)]
    let cpu_profile = match CpuProfileSession::start(&config) {
        Ok(session) => session,
        Err(e) => {
@@ -49,6 +57,8 @@ fn main() {
            std::process::exit(1);
        }
    };
+    #[cfg(not(unix))]
+    let cpu_profile: Option<()> = None;
    info!(
        "SeaweedFS Volume Server (Rust) v{}",
        seaweed_volume::version::full_version()
@@ -257,7 +267,7 @@ where

 async fn run(
    config: VolumeServerConfig,
-    cpu_profile: Option<CpuProfileSession>,
+    #[allow(unused_variables)] cpu_profile: CpuProfileParam,
 ) -> Result<(), Box<dyn std::error::Error>> {
    // Initialize the store
    let mut store = Store::new(config.index_type);
@@ -431,10 +441,12 @@ async fn run(
    }

    // Build HTTP routers
+    #[allow(unused_mut)]
    let mut admin_router = seaweed_volume::server::volume_server::build_admin_router_with_ui(
        state.clone(),
        config.ui_enabled,
    );
+    #[cfg(unix)]
    if config.pprof {
        admin_router = admin_router.merge(build_debug_router());
    }
@@ -588,31 +600,35 @@ async fn run(
        })
    };

+    // Bind the gRPC listener before spawning to propagate bind errors at startup.
+    let grpc_listener = tokio::net::TcpListener::bind(&grpc_addr)
+        .await
+        .unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
+    let grpc_local_addr = grpc_listener
+        .local_addr()
+        .unwrap_or_else(|e| panic!("Failed to get gRPC local addr: {}", e));
+
    let grpc_handle = {
        let grpc_state = state.clone();
-        let grpc_addr = grpc_addr.clone();
        let grpc_tls_acceptor = grpc_tls_acceptor.clone();
        let mut shutdown_rx = shutdown_tx.subscribe();
+        let shutdown_tx_grpc = shutdown_tx.clone();
        tokio::spawn(async move {
-            let addr = grpc_addr.parse().expect("Invalid gRPC address");
            let grpc_service = VolumeGrpcService {
                state: grpc_state.clone(),
            };
-            if let Some(tls_acceptor) = grpc_tls_acceptor {
-                let listener = tokio::net::TcpListener::bind(&grpc_addr)
-                    .await
-                    .unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
-                let incoming = grpc_tls_incoming(listener, tls_acceptor);
-                let reflection_v1 = tonic_reflection::server::Builder::configure()
-                    .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
-                    .build_v1()
-                    .expect("Failed to build gRPC reflection v1 service");
-                let reflection_v1alpha = tonic_reflection::server::Builder::configure()
-                    .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
-                    .build_v1alpha()
-                    .expect("Failed to build gRPC reflection v1alpha service");
-                info!("gRPC server listening on {} (TLS enabled)", addr);
-                if let Err(e) = build_grpc_server_builder()
+            let reflection_v1 = tonic_reflection::server::Builder::configure()
+                .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
+                .build_v1()
+                .expect("Failed to build gRPC reflection v1 service");
+            let reflection_v1alpha = tonic_reflection::server::Builder::configure()
+                .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
+                .build_v1alpha()
+                .expect("Failed to build gRPC reflection v1alpha service");
+            let result = if let Some(tls_acceptor) = grpc_tls_acceptor {
+                let incoming = grpc_tls_incoming(grpc_listener, tls_acceptor);
+                info!("gRPC server listening on {} (TLS enabled)", grpc_local_addr);
+                build_grpc_server_builder()
                    .layer(GrpcRequestIdLayer)
                    .add_service(reflection_v1)
                    .add_service(reflection_v1alpha)
@@ -621,32 +637,25 @@ async fn run(
                        let _ = shutdown_rx.recv().await;
                    })
                    .await
-                {
-                    error!("gRPC server error: {}", e);
-                }
            } else {
-                let reflection_v1 = tonic_reflection::server::Builder::configure()
-                    .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
-                    .build_v1()
-                    .expect("Failed to build gRPC reflection v1 service");
-                let reflection_v1alpha = tonic_reflection::server::Builder::configure()
-                    .register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
-                    .build_v1alpha()
-                    .expect("Failed to build gRPC reflection v1alpha service");
-                info!("gRPC server listening on {}", addr);
-                if let Err(e) = build_grpc_server_builder()
+                let incoming =
+                    tokio_stream::wrappers::TcpListenerStream::new(grpc_listener);
+                info!("gRPC server listening on {}", grpc_local_addr);
+                build_grpc_server_builder()
                    .layer(GrpcRequestIdLayer)
                    .add_service(reflection_v1)
                    .add_service(reflection_v1alpha)
                    .add_service(build_volume_grpc_service(grpc_service))
-                    .serve_with_shutdown(addr, async move {
+                    .serve_with_incoming_shutdown(incoming, async move {
                        let _ = shutdown_rx.recv().await;
                    })
                    .await
-                {
-                    error!("gRPC server error: {}", e);
-                }
+            };
+            if let Err(ref e) = result {
+                error!("gRPC server error: {}", e);
+                let _ = shutdown_tx_grpc.send(());
            }
+            result
        })
    };

@@ -721,6 +730,7 @@ async fn run(
        None
    };

+    #[cfg(unix)]
    let debug_handle = if config.debug {
        let debug_addr = format!("0.0.0.0:{}", config.debug_port);
        info!("Debug pprof server listening on {}", debug_addr);
@@ -742,6 +752,8 @@ async fn run(
    } else {
        None
    };
+    #[cfg(not(unix))]
+    let debug_handle: Option<tokio::task::JoinHandle<()>> = None;

    let metrics_push_handle = {
        let push_state = state.clone();
@@ -752,9 +764,40 @@ async fn run(
        }))
    };

-    // Wait for all servers
+    // Wait for servers. Use select! with &mut so the losing handle is not
+    // dropped, then await it explicitly afterward.
+    let mut server_err: Option<String> = None;
+    let mut http_handle = http_handle;
+    let mut grpc_handle = grpc_handle;
+    let grpc_finished_first = tokio::select! {
+        _ = &mut http_handle => false,
+        _ = &mut grpc_handle => true,
+    };
+    // Inspect the gRPC result (already resolved if it finished first,
+    // otherwise await it now).
+    let grpc_result = if grpc_finished_first {
+        grpc_handle.await
+    } else {
+        // HTTP finished first; gRPC is still running. Await it.
+        grpc_handle.await
+    };
+    match grpc_result {
+        Ok(Ok(())) => {}
+        Ok(Err(e)) => {
+            let msg = format!("gRPC server exited with error: {}", e);
+            error!("{}", msg);
+            server_err = Some(msg);
+            // serve error already sent shutdown inside the task
+        }
+        Err(e) => {
+            let msg = format!("gRPC task panicked: {}", e);
+            error!("{}", msg);
+            server_err = Some(msg);
+            let _ = shutdown_tx.send(());
+        }
+    }
+    // Ensure the HTTP handle completes too.
    let _ = http_handle.await;
-    let _ = grpc_handle.await;
    if let Some(h) = public_handle {
        let _ = h.await;
    }
@@ -774,10 +817,15 @@ async fn run(
    // Close all volumes (flush and release file handles) matching Go's Shutdown()
    state.store.write().unwrap().close();

+    #[cfg(unix)]
    if let Some(cpu_profile) = cpu_profile {
        cpu_profile.finish().map_err(std::io::Error::other)?;
    }

+    if let Some(err_msg) = server_err {
+        return Err(std::io::Error::other(err_msg).into());
+    }
+
    info!("Volume server stopped.");
    Ok(())
 }
@@ -1867,6 +1867,7 @@ impl VolumeServer for VolumeGrpcService {
        {
            let needle_header = resp.needle_header;
            let mut needle_body = resp.needle_body;
+            let resp_version = resp.version;

            if needle_header.is_empty() {
                continue;
@@ -1891,8 +1892,36 @@ impl VolumeServer for VolumeGrpcService {
            // Parse needle from header + body
            let mut n = Needle::default();
            n.read_header(&needle_header);
-            n.read_body_v2(&needle_body)
-                .map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
+
+            if n.size.0 < 0 {
+                return Err(Status::invalid_argument(format!(
+                    "unexpected negative needle size {} for needle {}",
+                    n.size.0, n.id.0
+                )));
+            } else if n.size.0 > 0 {
+                // Normal needle: parse the body fields (DataSize, Data, flags, etc.)
+                n.read_body_v2(&needle_body)
+                    .map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
+            } else {
+                // Delete tombstone (size == 0): body is checksum + timestamp
+                // (V3) or checksum only (V2) + padding. Validate minimum
+                // footer length for the protocol version.
+                use crate::storage::types::{
+                    NEEDLE_CHECKSUM_SIZE, TIMESTAMP_SIZE, VERSION_3, Version,
+                };
+                let version = Version(resp_version as u8);
+                let min_footer = if version >= VERSION_3 {
+                    NEEDLE_CHECKSUM_SIZE + TIMESTAMP_SIZE
+                } else {
+                    NEEDLE_CHECKSUM_SIZE
+                };
+                if needle_body.len() < min_footer {
+                    return Err(Status::invalid_argument(format!(
+                        "tombstone needle {} body too short: got {} bytes, need >= {} for version {}",
+                        n.id.0, needle_body.len(), min_footer, resp_version
+                    )));
+                }
+            }

            // Write needle to local volume
            let mut store = state.store.write().unwrap();
@@ -4047,11 +4076,12 @@ fn find_last_append_at_ns(idx_path: &str, dat_path: &str, version: u32) -> Optio
    let mut header = [0u8; 16];
    dat_file.read_exact(&mut header).ok()?;
    let needle_size = i32::from_be_bytes([header[12], header[13], header[14], header[15]]);
-    if needle_size <= 0 {
+    if needle_size < 0 {
        return None;
    }

    // Seek to tail: offset + 16 (header) + size -> checksum (4) + timestamp (8)
+    // For delete needles (size == 0), the tail is right after the header.
    let tail_offset = actual_offset as u64 + 16 + needle_size as u64;
    dat_file.seek(SeekFrom::Start(tail_offset)).ok()?;

@@ -1,9 +1,11 @@
+#[cfg(unix)]
 pub mod debug;
 pub mod grpc_client;
 pub mod grpc_server;
 pub mod handlers;
 pub mod heartbeat;
 pub mod memory_status;
+#[cfg(unix)]
 pub mod profiling;
 pub mod request_id;
 pub mod server_stats;
@@ -6,7 +6,7 @@
 use std::fs::File;
 use std::io;
 #[cfg(not(unix))]
-use std::io::{Seek, SeekFrom};
+use std::io::{Read, Seek, SeekFrom};

 use reed_solomon_erasure::galois_8::ReedSolomon;

@@ -249,11 +249,18 @@ func testConcurrentDirectoryOperations(t *testing.T, framework *FuseTestFramewor
 					return
 				}

-				// Create file in subdirectory
+				// Create file in subdirectory with retry for transient FUSE errors
 				testFile := filepath.Join(subDir, "test.txt")
 				content := []byte(fmt.Sprintf("Worker %d, Subdir %d", workerID, i))
-				if err := os.WriteFile(testFile, content, 0644); err != nil {
-					addError(fmt.Errorf("worker %d file %d: %v", workerID, i, err))
+				var writeErr error
+				for attempt := 0; attempt < 3; attempt++ {
+					if writeErr = os.WriteFile(testFile, content, 0644); writeErr == nil {
+						break
+					}
+					time.Sleep(100 * time.Millisecond)
+				}
+				if writeErr != nil {
+					addError(fmt.Errorf("worker %d file %d: %v", workerID, i, writeErr))
 					return
 				}
 			}
@@ -85,9 +85,19 @@ func testGitCloneAndPull(t *testing.T, mountPoint, localDir string) {
 	branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
 	gitRun(t, localClone, "push", "origin", branch)

+	// The bare repo lives on the FUSE mount and can briefly disappear after
+	// the push completes, and pushed pack objects may not be immediately
+	// consistent on the FUSE layer. Give the mount a chance to settle, then
+	// recover from the local clone if the remote is still missing.
+	if !waitForBareRepoEventually(t, bareRepo, 10*time.Second) {
+		t.Logf("bare repo %s did not stabilise after push; forcing recovery before clone", bareRepo)
+	}
+	refreshDirEntry(t, bareRepo)
+	time.Sleep(1 * time.Second)
+
 	// ---- Phase 3: Clone from mount bare repo into on-mount working dir ----
 	t.Log("Phase 3: clone from mount bare repo to on-mount working dir")
-	gitRun(t, "", "clone", bareRepo, mountClone)
+	ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)

 	assertFileContains(t, filepath.Join(mountClone, "README.md"), "# Updated")
 	assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v2")
@@ -290,7 +300,7 @@ func waitForBareRepoEventually(t *testing.T, bareRepo string, timeout time.Durat
 	t.Helper()
 	deadline := time.Now().Add(timeout)
 	for time.Now().Before(deadline) {
-		if isBareRepo(bareRepo) {
+		if isBareRepoAccessible(bareRepo) {
 			return true
 		}
 		refreshDirEntry(t, bareRepo)
@@ -312,6 +322,14 @@ func isBareRepo(bareRepo string) bool {
 	return true
 }

+func isBareRepoAccessible(bareRepo string) bool {
+	if !isBareRepo(bareRepo) {
+		return false
+	}
+	out, err := tryGitCommand("", "--git-dir="+bareRepo, "rev-parse", "--is-bare-repository")
+	return err == nil && out == "true"
+}
+
 func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
 	t.Helper()
 	require.NoError(t, tryEnsureMountClone(bareRepo, mountClone))
@@ -320,7 +338,7 @@ func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
 // tryEnsureBareRepo verifies the bare repo on the FUSE mount exists.
 // If it has vanished, it re-creates it from the local clone.
 func tryEnsureBareRepo(bareRepo, localClone string) error {
-	if _, err := os.Stat(filepath.Join(bareRepo, "HEAD")); err == nil {
+	if isBareRepoAccessible(bareRepo) {
 		return nil
 	}
 	branch, err := tryGitCommand(localClone, "rev-parse", "--abbrev-ref", "HEAD")
@@ -342,6 +360,36 @@ func tryEnsureBareRepo(bareRepo, localClone string) error {
 	return nil
 }

+func ensureMountCloneFromBareWithRecovery(t *testing.T, bareRepo, localClone, mountClone string) {
+	t.Helper()
+	const maxAttempts = 3
+	var lastErr error
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		if lastErr = tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone); lastErr == nil {
+			return
+		}
+		if attempt == maxAttempts {
+			require.NoError(t, lastErr, "git clone %s %s failed after %d recovery attempts", bareRepo, mountClone, maxAttempts)
+		}
+		t.Logf("clone recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
+		os.RemoveAll(mountClone)
+		time.Sleep(2 * time.Second)
+	}
+}
+
+func tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone string) error {
+	if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
+		return fmt.Errorf("ensure bare repo: %w", err)
+	}
+	if err := tryEnsureMountClone(bareRepo, mountClone); err != nil {
+		return fmt.Errorf("ensure mount clone: %w", err)
+	}
+	if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
+		return fmt.Errorf("verify mount clone: %w", err)
+	}
+	return nil
+}
+
 // tryEnsureMountClone is like ensureMountClone but returns an error instead
 // of failing the test, for use in recovery loops.
 func tryEnsureMountClone(bareRepo, mountClone string) error {
@@ -442,6 +490,20 @@ func resetToCommitWithRecovery(t *testing.T, bareRepo, localClone, mountClone, c
 			}
 			continue
 		}
+		// The kernel dcache can drop the FUSE entry moments after a
+		// successful check.  Wait briefly and re-verify to confirm the
+		// directory has stabilised before returning to the caller.
+		time.Sleep(1 * time.Second)
+		refreshDirEntry(t, mountClone)
+		if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
+			lastErr = fmt.Errorf("post-reset stabilisation check failed: %w", err)
+			if attempt < maxAttempts {
+				t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
+				os.RemoveAll(mountClone)
+				time.Sleep(2 * time.Second)
+			}
+			continue
+		}
 		return
 	}
 	require.NoError(t, lastErr, "git reset --hard %s failed after %d recovery attempts", commit, maxAttempts)
@@ -469,6 +531,10 @@ func tryPullFromCommit(t *testing.T, bareRepo, localClone, cloneDir, fromCommit
 	}
 	refreshDirEntry(t, cloneDir)

+	// Let the dcache stabilise before proceeding.
+	time.Sleep(1 * time.Second)
+	refreshDirEntry(t, cloneDir)
+
 	head, err := tryGitCommand(cloneDir, "rev-parse", "HEAD")
 	if err != nil {
 		return fmt.Errorf("rev-parse after reset: %w", err)
@@ -532,3 +598,33 @@ func TestTryEnsureBareRepoPreservesCurrentBranch(t *testing.T) {
 	assert.Equal(t, branch, restoredHead, "clone from recovered bare repo should check out the current branch")
 	assertFileContains(t, filepath.Join(restoredClone, "README.md"), "hello recovery")
 }
+
+func TestEnsureMountCloneFromBareWithRecoveryRecreatesMissingBareRepo(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "git_mount_clone_recovery_")
+	require.NoError(t, err)
+	defer os.RemoveAll(tempDir)
+
+	bareRepo := filepath.Join(tempDir, "repo.git")
+	localClone := filepath.Join(tempDir, "clone")
+	mountClone := filepath.Join(tempDir, "mount-clone")
+
+	gitRun(t, "", "init", "--bare", bareRepo)
+	gitRun(t, "", "clone", bareRepo, localClone)
+	gitRun(t, localClone, "config", "user.email", "test@seaweedfs.test")
+	gitRun(t, localClone, "config", "user.name", "Test")
+
+	writeFile(t, localClone, "README.md", "hello clone recovery\n")
+	gitRun(t, localClone, "add", "README.md")
+	gitRun(t, localClone, "commit", "-m", "initial commit")
+
+	branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
+	gitRun(t, localClone, "push", "origin", branch)
+
+	require.NoError(t, os.RemoveAll(bareRepo))
+
+	ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)
+
+	head := gitOutput(t, mountClone, "rev-parse", "--abbrev-ref", "HEAD")
+	assert.Equal(t, branch, head, "recovered clone should stay on the pushed branch")
+	assertFileContains(t, filepath.Join(mountClone, "README.md"), "hello clone recovery")
+}
@@ -14,7 +14,7 @@ require (
 replace github.com/seaweedfs/seaweedfs => ../../

 require (
-	cloud.google.com/go/auth v0.17.0 // indirect
+	cloud.google.com/go/auth v0.18.1 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
@@ -43,25 +43,25 @@ require (
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/aws/aws-sdk-go v1.55.8 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.41.4 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.32.7 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.32.13 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.13 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect
 	github.com/aws/smithy-go v1.24.2 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -104,7 +104,7 @@ require (
 	github.com/go-chi/chi/v5 v5.2.5 // indirect
 	github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
 	github.com/go-git/go-billy/v5 v5.6.2 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
@@ -124,8 +124,8 @@ require (
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
-	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
+	github.com/googleapis/gax-go/v2 v2.17.0 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gorilla/schema v1.4.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -147,9 +147,9 @@ require (
 	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
 	github.com/karlseguin/ccache/v2 v2.0.8 // indirect
-	github.com/klauspost/compress v1.18.4 // indirect
+	github.com/klauspost/compress v1.18.5 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
-	github.com/klauspost/reedsolomon v1.13.0 // indirect
+	github.com/klauspost/reedsolomon v1.13.3 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
@@ -237,7 +237,7 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
-	github.com/zeebo/xxh3 v1.0.2 // indirect
+	github.com/zeebo/xxh3 v1.1.0 // indirect
 	go.etcd.io/bbolt v1.4.3 // indirect
 	go.mongodb.org/mongo-driver v1.17.9 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
@@ -247,18 +247,18 @@ require (
 	go.opentelemetry.io/otel/trace v1.40.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.48.0 // indirect
-	golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
-	golang.org/x/image v0.36.0 // indirect
-	golang.org/x/net v0.49.0 // indirect
+	golang.org/x/crypto v0.49.0 // indirect
+	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
+	golang.org/x/image v0.38.0 // indirect
+	golang.org/x/net v0.51.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.42.0 // indirect
-	golang.org/x/term v0.40.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
+	golang.org/x/term v0.41.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
-	google.golang.org/api v0.258.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
+	google.golang.org/api v0.267.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
 	google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
@@ -13,8 +13,8 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
 cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4=
-cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ=
+cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
+cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
@@ -112,44 +112,44 @@ github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc h1:LoL75er
 github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc/go.mod h1:w648aMHEgFYS6xb0KVMMtZ2uMeemhiKCuD2vj6gY52A=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
-github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
+github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
+github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
-github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
+github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
+github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZifkk6vEGl/1Qor0=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
 github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
 github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
@@ -284,8 +284,8 @@ github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
-github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
+github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -385,12 +385,12 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
 github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ=
-github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
+github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
+github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
-github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
+github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
+github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
 github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
 github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
@@ -467,12 +467,12 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt
 github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
-github.com/klauspost/reedsolomon v1.13.0 h1:E0Cmgf2kMuhZTj6eefnvpKC4/Q4jhCi9YIjcZjK4arc=
-github.com/klauspost/reedsolomon v1.13.0/go.mod h1:ggJT9lc71Vu+cSOPBlxGvBN6TfAS77qB4fp8vJ05NSA=
+github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
+github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 h1:CjEMN21Xkr9+zwPmZPaJJw+apzVbjGL5uK/6g9Q2jGU=
 github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988/go.mod h1:/agobYum3uo/8V6yPVnq+R82pyVGCeuWW5arT4Txn8A=
 github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 h1:FHVoZMOVRA+6/y4yRlbiR3WvsrOcKBd/f64H7YiWR2U=
@@ -707,8 +707,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
-github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
+github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -739,8 +739,8 @@ github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
 github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
 github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
 github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
-github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
-github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
+github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
+github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
 go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
 go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
@@ -787,8 +787,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
-golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
+golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -799,12 +799,12 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
-golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
+golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
+golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
-golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
+golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
+golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -869,8 +869,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
-golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
+golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
+golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -894,8 +894,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -959,8 +959,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
-golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
-golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
+golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
+golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -976,8 +976,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1030,14 +1030,14 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
-golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
+golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
+golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
-gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -1054,8 +1054,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.258.0 h1:IKo1j5FBlN74fe5isA2PVozN3Y5pwNKriEgAXPOkDAc=
-google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww=
+google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
+google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1091,12 +1091,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 h1:dDbsTLIK7EzwUq36kCSAsk0slouq/S0tWHeeGi97cD8=
-google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846/go.mod h1:PP0g88Dz3C7hRAfbQCQggeWAXjuqGsNPLE4s7jh0RGU=
-google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
-google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
+google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
+google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
+google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
+google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -11,9 +11,11 @@ import (
 	"testing"
 	"time"

+	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 )

 // VolumeServer provides a minimal volume server for erasure coding tests.
@@ -196,12 +198,25 @@ func (v *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream vo
 	defer file.Close()

 	buf := make([]byte, 64*1024)
+	remaining := int64(req.GetStopOffset())
 	for {
-		n, readErr := file.Read(buf)
+		if remaining == 0 {
+			break
+		}
+
+		readBuf := buf
+		if remaining > 0 && remaining < int64(len(buf)) {
+			readBuf = buf[:remaining]
+		}
+
+		n, readErr := file.Read(readBuf)
 		if n > 0 {
-			if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: buf[:n]}); err != nil {
+			if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: readBuf[:n]}); err != nil {
 				return err
 			}
+			if remaining > 0 {
+				remaining -= int64(n)
+			}
 		}
 		if readErr == io.EOF {
 			break
@@ -307,10 +322,21 @@ func (v *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_ser
 	v.mu.Lock()
 	v.readFileStatusCalls++
 	v.mu.Unlock()
+
+	datInfo, err := os.Stat(v.filePath(req.VolumeId, ".dat"))
+	if err != nil {
+		return nil, err
+	}
+
+	idxInfo, err := os.Stat(v.filePath(req.VolumeId, ".idx"))
+	if err != nil {
+		return nil, err
+	}
+
 	return &volume_server_pb.ReadVolumeFileStatusResponse{
 		VolumeId:    req.VolumeId,
-		DatFileSize: 1024,
-		IdxFileSize: 16,
+		DatFileSize: uint64(datInfo.Size()),
+		IdxFileSize: uint64(idxInfo.Size()),
 		FileCount:   1,
 	}, nil
 }
@@ -349,7 +375,27 @@ func (v *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, strea
 	v.volumeCopyCalls++
 	v.mu.Unlock()

-	if err := stream.Send(&volume_server_pb.VolumeCopyResponse{ProcessedBytes: 1024}); err != nil {
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	var statusResp *volume_server_pb.ReadVolumeFileStatusResponse
+	if err := operation.WithVolumeServerClient(false, pb.ServerAddress(req.SourceDataNode), dialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			var readErr error
+			statusResp, readErr = client.ReadVolumeFileStatus(stream.Context(), &volume_server_pb.ReadVolumeFileStatusRequest{
+				VolumeId: req.VolumeId,
+			})
+			return readErr
+		}); err != nil {
+		return err
+	}
+
+	if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".dat", statusResp.DatFileSize, dialOption); err != nil {
+		return err
+	}
+	if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".idx", statusResp.IdxFileSize, dialOption); err != nil {
+		return err
+	}
+
+	if err := stream.Send(&volume_server_pb.VolumeCopyResponse{ProcessedBytes: int64(statusResp.DatFileSize + statusResp.IdxFileSize)}); err != nil {
 		return err
 	}
 	return stream.Send(&volume_server_pb.VolumeCopyResponse{LastAppendAtNs: uint64(time.Now().UnixNano())})
@@ -368,3 +414,44 @@ func (v *VolumeServer) VolumeTailReceiver(ctx context.Context, req *volume_serve
 	v.mu.Unlock()
 	return &volume_server_pb.VolumeTailReceiverResponse{}, nil
 }
+
+func (v *VolumeServer) copyRemoteFile(ctx context.Context, sourceDataNode string, volumeID uint32, ext string, fileSize uint64, dialOption grpc.DialOption) error {
+	path := v.filePath(volumeID, ext)
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return err
+	}
+
+	file, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	return operation.WithVolumeServerClient(true, pb.ServerAddress(sourceDataNode), dialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			stream, err := client.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+				VolumeId:   volumeID,
+				Ext:        ext,
+				StopOffset: fileSize,
+			})
+			if err != nil {
+				return err
+			}
+
+			for {
+				resp, recvErr := stream.Recv()
+				if recvErr == io.EOF {
+					return nil
+				}
+				if recvErr != nil {
+					return recvErr
+				}
+				if len(resp.FileContent) == 0 {
+					continue
+				}
+				if _, err := file.Write(resp.FileContent); err != nil {
+					return err
+				}
+			}
+		})
+}
@@ -16,6 +16,8 @@ import (
 	"google.golang.org/protobuf/proto"
 )

+const testVolumeDatSize = 1 * 1024 * 1024
+
 func TestVolumeBalanceExecutionIntegration(t *testing.T) {
 	volumeID := uint32(303)

@@ -31,6 +33,7 @@ func TestVolumeBalanceExecutionIntegration(t *testing.T) {

 	source := pluginworkers.NewVolumeServer(t, "")
 	target := pluginworkers.NewVolumeServer(t, "")
+	pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), volumeID, testVolumeDatSize)

 	job := &plugin_pb.JobSpec{
 		JobId:   fmt.Sprintf("balance-job-%d", volumeID),
@@ -84,6 +87,9 @@ func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {

 	// Build a batch job with 3 volume moves from source → target.
 	volumeIDs := []uint32{401, 402, 403}
+	for _, vid := range volumeIDs {
+		pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), vid, testVolumeDatSize)
+	}
 	moves := make([]*worker_pb.BalanceMoveSpec, len(volumeIDs))
 	for i, vid := range volumeIDs {
 		moves[i] = &worker_pb.BalanceMoveSpec{
@@ -139,10 +145,11 @@ func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
 		require.True(t, deletedVols[vid], "volume %d should have been deleted from source", vid)
 	}

-	// Pre-delete verification should have called ReadVolumeFileStatus on both
-	// source and target for each volume.
-	require.Equal(t, len(volumeIDs), source.ReadFileStatusCount(),
-		"each move should read source volume status before delete")
+	// Each move reads source status once before copy and once inside the
+	// target's fake VolumeCopy implementation, then reads target status once
+	// before deleting the source.
+	require.Equal(t, len(volumeIDs)*2, source.ReadFileStatusCount(),
+		"each move should read source volume status before copy and during target copy")
 	require.Equal(t, len(volumeIDs), target.ReadFileStatusCount(),
 		"each move should read target volume status before delete")

@@ -21,7 +21,9 @@ import (
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc"
@@ -139,6 +141,7 @@ func startDistributedLockCluster(t *testing.T) *distributedLockCluster {
 		require.NoError(t, cluster.waitForTCP(cluster.filerGRPCAddress(i), 30*time.Second), "wait for filer %d grpc\n%s", i, cluster.tailLog(fmt.Sprintf("filer%d.log", i)))
 	}
 	require.NoError(t, cluster.waitForFilerCount(2, 30*time.Second), "wait for filer group registration")
+	require.NoError(t, cluster.waitForLockRingConverged(30*time.Second), "wait for lock ring convergence")

 	for i := 0; i < 2; i++ {
 		require.NoError(t, cluster.startS3(i), "start s3 %d", i)
@@ -411,6 +414,130 @@ func (c *distributedLockCluster) waitForFilerCount(expected int, timeout time.Du
 	return fmt.Errorf("timed out waiting for %d filers in group %q", expected, c.filerGroup)
 }

+// waitForLockRingConverged verifies that both filers have a consistent view of the
+// lock ring by acquiring the same lock through each filer and checking mutual exclusion.
+// This guards against the window between master seeing both filers and the filers
+// actually receiving the LockRingUpdate broadcast (delayed by the stabilization timer).
+//
+// A single arbitrary key could false-pass: if the key's real primary is filer0 and
+// filer0 has a stale ring (only sees itself), it still grants correctly because it IS
+// the primary. To catch the stale ring we must also test a key whose real primary is
+// filer1. So we generate one test key per primary filer using the same consistent-hash
+// ring as production, and require mutual exclusion for all of them.
+func (c *distributedLockCluster) waitForLockRingConverged(timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+
+	owners := make([]pb.ServerAddress, 0, len(c.filerPorts))
+	for i := range c.filerPorts {
+		owners = append(owners, c.filerServerAddress(i))
+	}
+	ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	ring.SetServers(owners)
+
+	attempt := 0
+	for time.Now().Before(deadline) {
+		// Generate one unique test key per primary filer
+		testKeys := c.convergenceKeysPerPrimary(ring, owners, attempt)
+		attempt++
+
+		allConverged := true
+		for _, key := range testKeys {
+			converged, _ := c.checkLockMutualExclusion(key)
+			if !converged {
+				allConverged = false
+				break
+			}
+		}
+		if allConverged {
+			return nil
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("lock ring did not converge: both filers independently grant the same lock")
+}
+
+// convergenceKeysPerPrimary returns one lock key per distinct primary filer,
+// using the same consistent-hash ring as production routing.
+func (c *distributedLockCluster) convergenceKeysPerPrimary(ring *lock_manager.HashRing, owners []pb.ServerAddress, attempt int) []string {
+	keysByOwner := make(map[pb.ServerAddress]string, len(owners))
+	for i := 0; len(keysByOwner) < len(owners) && i < 1024; i++ {
+		candidate := fmt.Sprintf("convergence-%d-%d", attempt, i)
+		primary := ring.GetPrimary(candidate)
+		if _, exists := keysByOwner[primary]; !exists {
+			keysByOwner[primary] = candidate
+		}
+	}
+	keys := make([]string, 0, len(keysByOwner))
+	for _, k := range keysByOwner {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+// checkLockMutualExclusion acquires a lock via filer0, then tries the same lock via filer1.
+// Returns true if mutual exclusion holds (second attempt is denied).
+func (c *distributedLockCluster) checkLockMutualExclusion(testKey string) (bool, error) {
+	conn0, err := grpc.NewClient(c.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false, err
+	}
+	defer conn0.Close()
+
+	conn1, err := grpc.NewClient(c.filerGRPCAddress(1), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false, err
+	}
+	defer conn1.Close()
+
+	client0 := filer_pb.NewSeaweedFilerClient(conn0)
+	client1 := filer_pb.NewSeaweedFilerClient(conn1)
+
+	// Acquire lock via filer0
+	ctx0, cancel0 := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel0()
+	resp0, err := client0.DistributedLock(ctx0, &filer_pb.LockRequest{
+		Name:          testKey,
+		SecondsToLock: 5,
+		Owner:         "convergence-filer0",
+	})
+	if err != nil || resp0.RenewToken == "" {
+		return false, fmt.Errorf("filer0 lock failed: err=%v resp=%v", err, resp0)
+	}
+	defer func() {
+		// Always release the lock we acquired
+		client0.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
+			Name:       testKey,
+			RenewToken: resp0.RenewToken,
+		})
+	}()
+
+	// Try the same lock via filer1 - should be denied
+	ctx1, cancel1 := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel1()
+	resp1, err := client1.DistributedLock(ctx1, &filer_pb.LockRequest{
+		Name:          testKey,
+		SecondsToLock: 5,
+		Owner:         "convergence-filer1",
+	})
+	if err != nil {
+		return false, err
+	}
+	if resp1.RenewToken != "" {
+		// Both filers granted the lock - ring not converged. Release the second lock.
+		client1.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
+			Name:       testKey,
+			RenewToken: resp1.RenewToken,
+		})
+		return false, nil
+	}
+	// Verify the denial is specifically because the lock is already held,
+	// not due to a transient error that might give a false positive.
+	if !strings.Contains(resp1.Error, "lock already owned") {
+		return false, nil
+	}
+	return true, nil
+}
+
 func (c *distributedLockCluster) waitForHTTP(url string, timeout time.Duration) error {
 	client := &net.Dialer{Timeout: time.Second}
 	httpClient := &httpClientWithDialer{dialer: client}
@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"sort"
 	"strings"
 	"sync"
 	"testing"
@@ -15,9 +14,9 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/smithy-go"
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -149,14 +148,15 @@ func (c *distributedLockCluster) findLockOwnerKeys(bucket, prefix string) map[pb
 	for i := range c.filerPorts {
 		owners = append(owners, c.filerServerAddress(i))
 	}
-	sort.Slice(owners, func(i, j int) bool {
-		return owners[i] < owners[j]
-	})
+
+	ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	ring.SetServers(owners)

 	keysByOwner := make(map[pb.ServerAddress]string, len(owners))
 	for i := 0; i < 1024 && len(keysByOwner) < len(owners); i++ {
 		key := fmt.Sprintf("%s-%03d.txt", prefix, i)
-		lockOwner := ownerForObjectLock(bucket, key, owners)
+		lockKey := fmt.Sprintf("s3.object.write:/buckets/%s/%s", bucket, s3_constants.NormalizeObjectKey(key))
+		lockOwner := ring.GetPrimary(lockKey)
 		if _, exists := keysByOwner[lockOwner]; !exists {
 			keysByOwner[lockOwner] = key
 		}
@@ -164,15 +164,6 @@ func (c *distributedLockCluster) findLockOwnerKeys(bucket, prefix string) map[pb
 	return keysByOwner
 }

-func ownerForObjectLock(bucket, object string, owners []pb.ServerAddress) pb.ServerAddress {
-	lockKey := fmt.Sprintf("s3.object.write:/buckets/%s/%s", bucket, s3_constants.NormalizeObjectKey(object))
-	hash := util.HashStringToLong(lockKey)
-	if hash < 0 {
-		hash = -hash
-	}
-	return owners[hash%int64(len(owners))]
-}
-
 func lockOwnerLabel(owner pb.ServerAddress) string {
 	replacer := strings.NewReplacer(":", "_", ".", "_")
 	return "owner_" + replacer.Replace(string(owner))
@@ -0,0 +1,511 @@
+package iam
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// GetFederationTokenTestResponse represents the STS GetFederationToken response
+type GetFederationTokenTestResponse struct {
+	XMLName xml.Name `xml:"GetFederationTokenResponse"`
+	Result  struct {
+		Credentials struct {
+			AccessKeyId     string `xml:"AccessKeyId"`
+			SecretAccessKey string `xml:"SecretAccessKey"`
+			SessionToken    string `xml:"SessionToken"`
+			Expiration      string `xml:"Expiration"`
+		} `xml:"Credentials"`
+		FederatedUser struct {
+			FederatedUserId string `xml:"FederatedUserId"`
+			Arn             string `xml:"Arn"`
+		} `xml:"FederatedUser"`
+	} `xml:"GetFederationTokenResult"`
+}
+
+func getTestCredentials() (string, string) {
+	accessKey := os.Getenv("STS_TEST_ACCESS_KEY")
+	if accessKey == "" {
+		accessKey = "admin"
+	}
+	secretKey := os.Getenv("STS_TEST_SECRET_KEY")
+	if secretKey == "" {
+		secretKey = "admin"
+	}
+	return accessKey, secretKey
+}
+
+// isGetFederationTokenImplemented checks if the running server supports GetFederationToken
+func isGetFederationTokenImplemented(t *testing.T) bool {
+	accessKey, secretKey := getTestCredentials()
+	resp, err := callSTSAPIWithSigV4(t, url.Values{
+		"Action":  {"GetFederationToken"},
+		"Version": {"2011-06-15"},
+		"Name":    {"probe"},
+	}, accessKey, secretKey)
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+	var errResp STSErrorTestResponse
+	if xml.Unmarshal(body, &errResp) == nil {
+		if errResp.Error.Code == "InvalidAction" || errResp.Error.Code == "NotImplemented" {
+			return false
+		}
+	}
+	return true
+}
+
+// TestSTSGetFederationTokenValidation tests input validation for the GetFederationToken endpoint
+func TestSTSGetFederationTokenValidation(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Fatal("SeaweedFS STS endpoint is not running at", TestSTSEndpoint, "- please run 'make setup-all-tests' first")
+	}
+
+	if !isGetFederationTokenImplemented(t) {
+		t.Fatal("GetFederationToken action is not implemented in the running server")
+	}
+
+	accessKey, secretKey := getTestCredentials()
+
+	t.Run("missing_name", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			// Name is missing
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "MissingParameter", errResp.Error.Code)
+	})
+
+	t.Run("name_too_short", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"A"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("name_too_long", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {strings.Repeat("A", 33)},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("name_invalid_characters", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"bad name"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("duration_too_short", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"GetFederationToken"},
+			"Version":         {"2011-06-15"},
+			"Name":            {"TestApp"},
+			"DurationSeconds": {"100"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("duration_too_long", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"GetFederationToken"},
+			"Version":         {"2011-06-15"},
+			"Name":            {"TestApp"},
+			"DurationSeconds": {"200000"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("malformed_policy", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"TestApp"},
+			"Policy":  {"not-valid-json"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "MalformedPolicyDocument", errResp.Error.Code)
+	})
+
+	t.Run("anonymous_rejected", func(t *testing.T) {
+		// GetFederationToken requires SigV4, anonymous should fail
+		resp, err := callSTSAPI(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"TestApp"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode)
+	})
+}
+
+// TestSTSGetFederationTokenRejectTemporaryCredentials tests that temporary
+// credentials (session tokens) are rejected by GetFederationToken
+func TestSTSGetFederationTokenRejectTemporaryCredentials(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	if !isGetFederationTokenImplemented(t) {
+		t.Skip("GetFederationToken not implemented")
+	}
+
+	accessKey, secretKey := getTestCredentials()
+
+	// First, obtain temporary credentials via AssumeRole
+	resp, err := callSTSAPIWithSigV4(t, url.Values{
+		"Action":          {"AssumeRole"},
+		"Version":         {"2011-06-15"},
+		"RoleArn":         {"arn:aws:iam::role/admin"},
+		"RoleSessionName": {"temp-session"},
+	}, accessKey, secretKey)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	if resp.StatusCode != http.StatusOK {
+		t.Skipf("AssumeRole failed (may not be configured): status=%d body=%s", resp.StatusCode, string(body))
+	}
+
+	var assumeResp AssumeRoleTestResponse
+	require.NoError(t, xml.Unmarshal(body, &assumeResp), "Parse AssumeRole response: %s", string(body))
+
+	tempAccessKey := assumeResp.Result.Credentials.AccessKeyId
+	tempSecretKey := assumeResp.Result.Credentials.SecretAccessKey
+	tempSessionToken := assumeResp.Result.Credentials.SessionToken
+	require.NotEmpty(t, tempAccessKey)
+	require.NotEmpty(t, tempSessionToken)
+
+	// Now try GetFederationToken with the temporary credentials
+	// Include X-Amz-Security-Token header which marks this as a temp credential call
+	params := url.Values{
+		"Action":  {"GetFederationToken"},
+		"Version": {"2011-06-15"},
+		"Name":    {"ShouldFail"},
+	}
+
+	reqBody := params.Encode()
+	req, err := http.NewRequest(http.MethodPost, TestSTSEndpoint+"/", strings.NewReader(reqBody))
+	require.NoError(t, err)
+
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("X-Amz-Security-Token", tempSessionToken)
+
+	creds := credentials.NewStaticCredentials(tempAccessKey, tempSecretKey, tempSessionToken)
+	signer := v4.NewSigner(creds)
+	_, err = signer.Sign(req, strings.NewReader(reqBody), "sts", "us-east-1", time.Now())
+	require.NoError(t, err)
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	resp2, err := client.Do(req)
+	require.NoError(t, err)
+	defer resp2.Body.Close()
+
+	body2, _ := io.ReadAll(resp2.Body)
+	assert.Equal(t, http.StatusForbidden, resp2.StatusCode,
+		"GetFederationToken should reject temporary credentials: %s", string(body2))
+	assert.Contains(t, string(body2), "temporary credentials",
+		"Error should mention temporary credentials")
+}
+
+// TestSTSGetFederationTokenSuccess tests a successful GetFederationToken call
+// and verifies the returned credentials can be used to access S3
+func TestSTSGetFederationTokenSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	if !isGetFederationTokenImplemented(t) {
+		t.Skip("GetFederationToken not implemented")
+	}
+
+	accessKey, secretKey := getTestCredentials()
+
+	t.Run("basic_success", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"AppClient"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
+
+		if resp.StatusCode != http.StatusOK {
+			var errResp STSErrorTestResponse
+			_ = xml.Unmarshal(body, &errResp)
+			t.Fatalf("GetFederationToken failed: code=%s message=%s", errResp.Error.Code, errResp.Error.Message)
+		}
+
+		var stsResp GetFederationTokenTestResponse
+		require.NoError(t, xml.Unmarshal(body, &stsResp), "Parse response: %s", string(body))
+
+		creds := stsResp.Result.Credentials
+		assert.NotEmpty(t, creds.AccessKeyId)
+		assert.NotEmpty(t, creds.SecretAccessKey)
+		assert.NotEmpty(t, creds.SessionToken)
+		assert.NotEmpty(t, creds.Expiration)
+
+		fedUser := stsResp.Result.FederatedUser
+		assert.Contains(t, fedUser.Arn, "federated-user/AppClient")
+		assert.Contains(t, fedUser.FederatedUserId, "AppClient")
+	})
+
+	t.Run("with_custom_duration", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"GetFederationToken"},
+			"Version":         {"2011-06-15"},
+			"Name":            {"DurationTest"},
+			"DurationSeconds": {"3600"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
+
+		if resp.StatusCode == http.StatusOK {
+			var stsResp GetFederationTokenTestResponse
+			require.NoError(t, xml.Unmarshal(body, &stsResp))
+			assert.NotEmpty(t, stsResp.Result.Credentials.AccessKeyId)
+
+			// Verify expiration is roughly 1 hour from now
+			expTime, err := time.Parse(time.RFC3339, stsResp.Result.Credentials.Expiration)
+			require.NoError(t, err)
+			diff := time.Until(expTime)
+			assert.InDelta(t, 3600, diff.Seconds(), 60,
+				"Expiration should be ~1 hour from now")
+		}
+	})
+
+	t.Run("with_36_hour_duration", func(t *testing.T) {
+		// GetFederationToken allows up to 36 hours (unlike AssumeRole's 12h max)
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"GetFederationToken"},
+			"Version":         {"2011-06-15"},
+			"Name":            {"LongDuration"},
+			"DurationSeconds": {"129600"}, // 36 hours
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		if resp.StatusCode == http.StatusOK {
+			var stsResp GetFederationTokenTestResponse
+			require.NoError(t, xml.Unmarshal(body, &stsResp))
+
+			expTime, err := time.Parse(time.RFC3339, stsResp.Result.Credentials.Expiration)
+			require.NoError(t, err)
+			diff := time.Until(expTime)
+			assert.InDelta(t, 129600, diff.Seconds(), 60,
+				"Expiration should be ~36 hours from now")
+		} else {
+			// Duration should not cause a rejection
+			var errResp STSErrorTestResponse
+			_ = xml.Unmarshal(body, &errResp)
+			assert.NotContains(t, errResp.Error.Message, "DurationSeconds",
+				"36-hour duration should be accepted by GetFederationToken")
+		}
+	})
+}
+
+// TestSTSGetFederationTokenWithSessionPolicy tests that vended credentials
+// are scoped down by an inline session policy
+func TestSTSGetFederationTokenWithSessionPolicy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	if !isGetFederationTokenImplemented(t) {
+		t.Skip("GetFederationToken not implemented")
+	}
+
+	accessKey, secretKey := getTestCredentials()
+
+	// Create a test bucket using admin credentials
+	adminSess, err := session.NewSession(&aws.Config{
+		Region:           aws.String("us-east-1"),
+		Endpoint:         aws.String(TestSTSEndpoint),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
+	})
+	require.NoError(t, err)
+
+	adminS3 := s3.New(adminSess)
+	bucket := fmt.Sprintf("fed-token-test-%d", time.Now().UnixNano())
+
+	_, err = adminS3.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
+	require.NoError(t, err)
+	defer adminS3.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)})
+
+	_, err = adminS3.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String("test.txt"),
+		Body:   strings.NewReader("hello"),
+	})
+	require.NoError(t, err)
+	defer adminS3.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String("test.txt")})
+
+	// Get federated credentials with a session policy that only allows GetObject
+	sessionPolicy := fmt.Sprintf(`{
+		"Version": "2012-10-17",
+		"Statement": [{
+			"Effect": "Allow",
+			"Action": ["s3:GetObject"],
+			"Resource": ["arn:aws:s3:::%s/*"]
+		}]
+	}`, bucket)
+
+	resp, err := callSTSAPIWithSigV4(t, url.Values{
+		"Action":  {"GetFederationToken"},
+		"Version": {"2011-06-15"},
+		"Name":    {"ScopedClient"},
+		"Policy":  {sessionPolicy},
+	}, accessKey, secretKey)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+	t.Logf("GetFederationToken response: status=%d body=%s", resp.StatusCode, string(body))
+
+	if resp.StatusCode != http.StatusOK {
+		t.Skipf("GetFederationToken failed (may need IAM policy config): %s", string(body))
+	}
+
+	var stsResp GetFederationTokenTestResponse
+	require.NoError(t, xml.Unmarshal(body, &stsResp))
+
+	fedCreds := stsResp.Result.Credentials
+	require.NotEmpty(t, fedCreds.AccessKeyId)
+	require.NotEmpty(t, fedCreds.SessionToken)
+
+	// Create S3 client with the federated credentials
+	fedSess, err := session.NewSession(&aws.Config{
+		Region:           aws.String("us-east-1"),
+		Endpoint:         aws.String(TestSTSEndpoint),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+		Credentials: credentials.NewStaticCredentials(
+			fedCreds.AccessKeyId, fedCreds.SecretAccessKey, fedCreds.SessionToken),
+	})
+	require.NoError(t, err)
+
+	fedS3 := s3.New(fedSess)
+
+	// GetObject should succeed (allowed by session policy)
+	getResp, err := fedS3.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String("test.txt"),
+	})
+	if err == nil {
+		defer getResp.Body.Close()
+		t.Log("GetObject with federated credentials succeeded (as expected)")
+	} else {
+		t.Logf("GetObject with federated credentials: %v (session policy enforcement may vary)", err)
+	}
+
+	// PutObject should be denied (not allowed by session policy)
+	_, err = fedS3.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String("denied.txt"),
+		Body:   strings.NewReader("should fail"),
+	})
+	if err != nil {
+		t.Log("PutObject correctly denied with federated credentials")
+		assert.Contains(t, err.Error(), "AccessDenied",
+			"PutObject should be denied by session policy")
+	} else {
+		// Clean up if unexpectedly succeeded
+		adminS3.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String("denied.txt")})
+		t.Log("PutObject unexpectedly succeeded — session policy enforcement may not be active")
+	}
+}
@@ -4,8 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"math/rand"
-	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -96,9 +94,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}

-	env.masterPort = mustFreePort(t, "Master")
-	env.filerPort = mustFreePort(t, "Filer")
-	env.s3Port = mustFreePort(t, "S3")
+	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
+	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
+	env.s3Port = testutil.MustFreeMiniPort(t, "S3")

 	bindIP := testutil.FindBindIP()
 	iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)
@@ -135,14 +133,14 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	}
 	registerMiniProcess(env.masterProcess)

-	if !waitForPort(env.masterPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
 	}
-	if !waitForPort(env.filerPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
 	}
-	if !waitForPort(env.s3Port, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
 	}
 }

@@ -208,46 +206,6 @@ func (env *TestEnvironment) Cleanup(t *testing.T) {
 	}
 }

-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-
-	for i := 0; i < 200; i++ {
-		port := 20000 + rand.Intn(30000)
-		listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port))
-		if err != nil {
-			continue
-		}
-		_ = listener.Close()
-
-		grpcPort := port + 10000
-		if grpcPort > 65535 {
-			continue
-		}
-		grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", grpcPort))
-		if err != nil {
-			continue
-		}
-		_ = grpcListener.Close()
-		return port
-	}
-
-	t.Fatalf("failed to get free port for %s", name)
-	return 0
-}
-
-func waitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
-		if err == nil {
-			_ = conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}
-
 func runSparkPyScript(t *testing.T, container testcontainers.Container, script string, s3Port int) (int, string) {
 	t.Helper()

@@ -98,8 +98,8 @@ start-seaweedfs: check-binary
 	# Create S3 configuration with SSE-KMS support
 	@printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}],"kms":{"type":"%s","configs":{"keyId":"%s","encryptionContext":{},"bucketKey":false}}}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" "$(KMS_TYPE)" "$(KMS_KEY_ID)" > /tmp/seaweedfs-sse-s3.json
 	
-	# Start weed mini
-	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(SEAWEEDFS_BINARY) mini \
+	# Start weed mini (WEED_S3_SSE_KEY enables SSE-S3 encryption)
+	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key $(SEAWEEDFS_BINARY) mini \
 		-dir=/tmp/seaweedfs-test-sse \
 		-s3.port=$(S3_PORT) \
 		-s3.config=/tmp/seaweedfs-sse-s3.json \
@@ -337,8 +337,9 @@ start-seaweedfs-ci: check-binary
 	     s3-config-template.json > /tmp/seaweedfs-s3.json
 	
 	# Start weed mini with embedded S3 using the JSON config (with verbose logging)
+	# WEED_S3_SSE_KEY enables SSE-S3 encryption for testing (KEK derived via HKDF)
 	@echo "Starting weed mini with embedded S3..."
-	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
+	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
 		-dir=/tmp/seaweedfs-test-sse \
 		-s3.port=$(S3_PORT) \
 		-s3.config=/tmp/seaweedfs-s3.json \
@@ -482,7 +483,7 @@ test-volume-encryption: build-weed
 	     -e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \
 	     s3-config-template.json > /tmp/seaweedfs-s3.json
 	@echo "Starting weed mini with S3 volume encryption..."
-	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
+	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
 		-dir=/tmp/seaweedfs-test-sse \
 		-s3.port=$(S3_PORT) \
 		-s3.config=/tmp/seaweedfs-s3.json \
@@ -289,6 +289,157 @@ func TestVersioningPaginationMultipleObjectsManyVersions(t *testing.T) {
 	})
 }

+// TestVersioningPaginationDeepDirectoryHierarchy tests that paginated ListObjectVersions
+// correctly skips directory subtrees before the key-marker. This reproduces the
+// real-world scenario where Veeam backup objects are spread across many subdirectories
+// (e.g., Mailboxes/<uuid>/ItemsData/<file>) and pagination becomes exponentially
+// slower as the marker advances through the tree.
+//
+// Run with: ENABLE_STRESS_TESTS=true go test -v -run TestVersioningPaginationDeepDirectoryHierarchy -timeout 10m
+func TestVersioningPaginationDeepDirectoryHierarchy(t *testing.T) {
+	if os.Getenv("ENABLE_STRESS_TESTS") != "true" {
+		t.Skip("Skipping stress test. Set ENABLE_STRESS_TESTS=true to run.")
+	}
+
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	enableVersioning(t, client, bucketName)
+	checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
+
+	// Create a deep directory structure mimicking Veeam 365 backup layout:
+	// Backup/Organizations/<org>/Mailboxes/<mailbox>/ItemsData/<file>
+	numMailboxes := 20
+	filesPerMailbox := 5
+	totalObjects := numMailboxes * filesPerMailbox
+	orgPrefix := "Backup/Organizations/org-001/Mailboxes"
+
+	t.Logf("Creating %d objects across %d subdirectories (depth=6)...", totalObjects, numMailboxes)
+	startTime := time.Now()
+
+	allKeys := make([]string, 0, totalObjects)
+	for i := 0; i < numMailboxes; i++ {
+		mailboxId := fmt.Sprintf("mbx-%03d", i)
+		for j := 0; j < filesPerMailbox; j++ {
+			key := fmt.Sprintf("%s/%s/ItemsData/file-%03d.dat", orgPrefix, mailboxId, j)
+			_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(key),
+				Body:   strings.NewReader(fmt.Sprintf("content-%d-%d", i, j)),
+			})
+			require.NoError(t, err)
+			allKeys = append(allKeys, key)
+		}
+	}
+	t.Logf("Created %d objects in %v", totalObjects, time.Since(startTime))
+
+	// Test 1: Paginate through all versions with a broad prefix and small maxKeys.
+	// This forces multiple pages that must skip earlier subdirectory trees.
+	t.Run("PaginateAcrossSubdirectories", func(t *testing.T) {
+		maxKeys := int32(10) // Force many pages to exercise marker skipping
+		var allVersions []types.ObjectVersion
+		var keyMarker, versionIdMarker *string
+		pageCount := 0
+		pageStartTimes := make([]time.Duration, 0)
+
+		for {
+			pageStart := time.Now()
+			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+				Bucket:          aws.String(bucketName),
+				Prefix:          aws.String(orgPrefix + "/"),
+				MaxKeys:         aws.Int32(maxKeys),
+				KeyMarker:       keyMarker,
+				VersionIdMarker: versionIdMarker,
+			})
+			pageDuration := time.Since(pageStart)
+			require.NoError(t, err)
+			pageCount++
+			pageStartTimes = append(pageStartTimes, pageDuration)
+
+			allVersions = append(allVersions, resp.Versions...)
+			t.Logf("Page %d: %d versions in %v (marker: %v)",
+				pageCount, len(resp.Versions), pageDuration,
+				keyMarker)
+
+			if resp.IsTruncated == nil || !*resp.IsTruncated {
+				break
+			}
+			keyMarker = resp.NextKeyMarker
+			versionIdMarker = resp.NextVersionIdMarker
+		}
+
+		assert.Greater(t, pageCount, 1, "Should require multiple pages")
+
+		// Verify listed keys exactly match the keys we created (same elements, same order)
+		listedKeys := make([]string, 0, len(allVersions))
+		for _, v := range allVersions {
+			listedKeys = append(listedKeys, *v.Key)
+		}
+		assert.Equal(t, allKeys, listedKeys,
+			"Listed version keys should exactly match created keys")
+
+		// Check that later pages don't take dramatically longer than earlier ones.
+		// Before the fix, later pages were exponentially slower because they
+		// re-traversed the entire tree. With the fix, all pages should be similar.
+		if len(pageStartTimes) >= 4 {
+			firstQuarter := pageStartTimes[0]
+			lastQuarter := pageStartTimes[len(pageStartTimes)-1]
+			t.Logf("First page: %v, Last page: %v, Ratio: %.1fx",
+				firstQuarter, lastQuarter,
+				float64(lastQuarter)/float64(firstQuarter))
+			// Allow generous 10x ratio to avoid flakiness; before the fix
+			// the ratio was 100x+ on large datasets
+			if lastQuarter > firstQuarter*10 && lastQuarter > 500*time.Millisecond {
+				t.Errorf("Last page took %.1fx longer than first page (%v vs %v) — possible pagination regression",
+					float64(lastQuarter)/float64(firstQuarter), lastQuarter, firstQuarter)
+			}
+		}
+	})
+
+	// Test 2: Paginate with delimiter to verify CommonPrefixes interaction
+	t.Run("PaginateWithDelimiterAcrossSubdirs", func(t *testing.T) {
+		maxKeys := int32(5)
+		var allPrefixes []string
+		var keyMarker *string
+		pageCount := 0
+
+		for {
+			resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+				Bucket:    aws.String(bucketName),
+				Prefix:    aws.String(orgPrefix + "/"),
+				Delimiter: aws.String("/"),
+				MaxKeys:   aws.Int32(maxKeys),
+				KeyMarker: keyMarker,
+			})
+			require.NoError(t, err)
+			pageCount++
+
+			for _, cp := range resp.CommonPrefixes {
+				allPrefixes = append(allPrefixes, *cp.Prefix)
+			}
+
+			if resp.IsTruncated == nil || !*resp.IsTruncated {
+				break
+			}
+			keyMarker = resp.NextKeyMarker
+		}
+
+		assert.Greater(t, pageCount, 1, "Should require multiple pages with maxKeys=%d", maxKeys)
+
+		// Build the exact expected prefixes
+		expectedPrefixes := make([]string, 0, numMailboxes)
+		for i := 0; i < numMailboxes; i++ {
+			expectedPrefixes = append(expectedPrefixes,
+				fmt.Sprintf("%s/mbx-%03d/", orgPrefix, i))
+		}
+		assert.Equal(t, expectedPrefixes, allPrefixes,
+			"CommonPrefixes should exactly match expected mailbox prefixes")
+	})
+}
+
 // listAllVersions is a helper to list all versions of a specific object using pagination
 func listAllVersions(t *testing.T, client *s3.Client, bucketName, objectKey string) []types.ObjectVersion {
 	var allVersions []types.ObjectVersion
@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -115,11 +114,11 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}

-	env.masterPort = mustFreePort(t, "Master")
-	env.filerPort = mustFreePort(t, "Filer")
-	env.s3Port = mustFreePort(t, "S3")
-	env.icebergRestPort = mustFreePort(t, "Iceberg")
-	env.risingwavePort = mustFreePort(t, "RisingWave")
+	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
+	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
+	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
+	env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")
+	env.risingwavePort = testutil.MustFreeMiniPort(t, "RisingWave")

 	env.bindIP = testutil.FindBindIP()

@@ -161,62 +160,20 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	registerMiniProcess(env.masterProcess)

 	// Wait for all services to be ready
-	if !waitForPort(env.masterPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
 	}
-	if !waitForPort(env.filerPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
 	}
-	if !waitForPort(env.s3Port, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
 	}
-	if !waitForPort(env.icebergRestPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - iceberg rest port %d not listening", env.icebergRestPort)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/v1/config", env.icebergRestPort), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - iceberg rest endpoint http://127.0.0.1:%d/v1/config not responding", env.icebergRestPort)
 	}
 }

-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-	minPort := 10000
-	maxPort := 55000 // Ensure port+10000 < 65535
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-
-	for i := 0; i < 1000; i++ {
-		port := minPort + r.Intn(maxPort-minPort)
-
-		// Check http port
-		ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
-		if err != nil {
-			continue
-		}
-		ln.Close()
-
-		// Check grpc port (weed mini uses port+10000)
-		ln2, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port+10000))
-		if err != nil {
-			continue
-		}
-		ln2.Close()
-
-		return port
-	}
-	t.Fatalf("failed to find a free port < %d for %s after 1000 attempts", maxPort, name)
-	return 0
-}
-
-func waitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
-		if err == nil {
-			conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}
-
 func (env *TestEnvironment) StartRisingWave(t *testing.T) {
 	t.Helper()

@@ -253,7 +210,7 @@ func (env *TestEnvironment) StartRisingWave(t *testing.T) {
 	}

 	// Wait for RisingWave port to be open on host
-	if !waitForPort(env.risingwavePort, 120*time.Second) {
+	if !testutil.WaitForPort(env.risingwavePort, 120*time.Second) {
 		t.Fatalf("timed out waiting for RisingWave port %d to be open", env.risingwavePort)
 	}

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
-	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -91,10 +90,10 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}

-	env.masterPort = mustFreePort(t, "Master")
-	env.filerPort = mustFreePort(t, "Filer")
-	env.s3Port = mustFreePort(t, "S3")
-	env.icebergRestPort = mustFreePort(t, "Iceberg")
+	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
+	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
+	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
+	env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")

 	bindIP := testutil.FindBindIP()

@@ -127,58 +126,20 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	registerMiniProcess(env.masterProcess)

 	// Wait for all services to be ready
-	if !waitForPort(env.masterPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
 	}
-	if !waitForPort(env.filerPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
 	}
-	if !waitForPort(env.s3Port, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
 	}
-	if !waitForPort(env.icebergRestPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - iceberg rest port %d not listening", env.icebergRestPort)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/v1/config", env.icebergRestPort), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - iceberg rest endpoint http://127.0.0.1:%d/v1/config not responding", env.icebergRestPort)
 	}
 }

-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-
-	for i := 0; i < 200; i++ {
-		port := 20000 + rand.Intn(30000)
-		listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port))
-		if err != nil {
-			continue
-		}
-		listener.Close()
-		grpcPort := port + 10000
-		if grpcPort > 65535 {
-			continue
-		}
-		grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", grpcPort))
-		if err != nil {
-			continue
-		}
-		grpcListener.Close()
-		return port
-	}
-	t.Fatalf("failed to get free port for %s", name)
-	return 0
-}
-
-func waitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
-		if err == nil {
-			conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}
-
 func (env *TestEnvironment) writeSparkConfig(t *testing.T, catalogBucket string) string {
 	t.Helper()

@@ -115,10 +115,13 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {

 	bindIP := testutil.FindBindIP()

-	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
-	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
-	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
-	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
+	// Allocate all ports in a single batch to prevent the OS from recycling
+	// a released port, which can cause two services to get the same port.
+	ports := testutil.MustAllocatePorts(t, 8)
+	masterPort, masterGrpcPort := ports[0], ports[1]
+	volumePort, volumeGrpcPort := ports[2], ports[3]
+	filerPort, filerGrpcPort := ports[4], ports[5]
+	s3Port, s3GrpcPort := ports[6], ports[7]

 	return &TestEnvironment{
 		seaweedDir:     seaweedDir,
@@ -89,11 +89,14 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {

 	bindIP := testutil.FindBindIP()

-	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
-	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
-	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
-	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
-	polarisPort, polarisAdminPort := testutil.MustFreePortPair(t, "Polaris")
+	// Allocate all ports in a single batch to prevent the OS from recycling
+	// a released port, which can cause two services to get the same port.
+	ports := testutil.MustAllocatePorts(t, 10)
+	masterPort, masterGrpcPort := ports[0], ports[1]
+	volumePort, volumeGrpcPort := ports[2], ports[3]
+	filerPort, filerGrpcPort := ports[4], ports[5]
+	s3Port, s3GrpcPort := ports[6], ports[7]
+	polarisPort, polarisAdminPort := ports[8], ports[9]

 	return &TestEnvironment{
 		seaweedDir:       seaweedDir,
@@ -93,10 +93,13 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {

 	bindIP := testutil.FindBindIP()

-	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
-	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
-	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
-	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3") // Changed to use testutil.MustFreePortPair
+	// Allocate all ports in a single batch to prevent the OS from recycling
+	// a released port, which can cause two services to get the same port.
+	ports := testutil.MustAllocatePorts(t, 8)
+	masterPort, masterGrpcPort := ports[0], ports[1]
+	volumePort, volumeGrpcPort := ports[2], ports[3]
+	filerPort, filerGrpcPort := ports[4], ports[5]
+	s3Port, s3GrpcPort := ports[6], ports[7]

 	return &TestEnvironment{
 		seaweedDir:      seaweedDir,
@@ -2,6 +2,7 @@ package testutil

 import (
 	"context"
+	"fmt"
 	"net"
 	"net/http"
 	"os/exec"
@@ -14,33 +15,44 @@ func HasDocker() bool {
 	return cmd.Run() == nil
 }

+// MustFreePortPair is a convenience wrapper for tests that only need a single pair.
+// Prefer MustAllocatePorts when allocating multiple pairs to guarantee uniqueness.
 func MustFreePortPair(t *testing.T, name string) (int, int) {
-	httpPort, grpcPort, err := findAvailablePortPair()
-	if err != nil {
-		t.Fatalf("Failed to get free port pair for %s: %v", name, err)
-	}
-	return httpPort, grpcPort
+	ports := MustAllocatePorts(t, 2)
+	return ports[0], ports[1]
 }

-func findAvailablePortPair() (int, int, error) {
-	httpPort, err := GetFreePort()
+// MustAllocatePorts allocates count unique free ports atomically.
+// All listeners are held open until every port is obtained, preventing
+// the OS from recycling a port between successive allocations.
+func MustAllocatePorts(t *testing.T, count int) []int {
+	t.Helper()
+	ports, err := AllocatePorts(count)
 	if err != nil {
-		return 0, 0, err
+		t.Fatalf("Failed to allocate %d free ports: %v", count, err)
 	}
-	grpcPort, err := GetFreePort()
-	if err != nil {
-		return 0, 0, err
-	}
-	return httpPort, grpcPort, nil
+	return ports
 }

-func GetFreePort() (int, error) {
-	listener, err := net.Listen("tcp", "0.0.0.0:0")
-	if err != nil {
-		return 0, err
+// AllocatePorts allocates count unique free ports atomically.
+func AllocatePorts(count int) ([]int, error) {
+	listeners := make([]net.Listener, 0, count)
+	ports := make([]int, 0, count)
+	for i := 0; i < count; i++ {
+		l, err := net.Listen("tcp", "0.0.0.0:0")
+		if err != nil {
+			for _, ll := range listeners {
+				_ = ll.Close()
+			}
+			return nil, err
+		}
+		listeners = append(listeners, l)
+		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
 	}
-	defer listener.Close()
-	return listener.Addr().(*net.TCPAddr).Port, nil
+	for _, l := range listeners {
+		_ = l.Close()
+	}
+	return ports, nil
 }

 func WaitForService(url string, timeout time.Duration) bool {
@@ -64,3 +76,17 @@ func WaitForService(url string, timeout time.Duration) bool {
 		}
 	}
 }
+
+func WaitForPort(port int, timeout time.Duration) bool {
+	deadline := time.Now().Add(timeout)
+	address := fmt.Sprintf("127.0.0.1:%d", port)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", address, 500*time.Millisecond)
+		if err == nil {
+			_ = conn.Close()
+			return true
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return false
+}
@@ -2,11 +2,16 @@ package testutil

 import (
 	"fmt"
+	"math/rand"
 	"net"
 	"os"
 	"path/filepath"
+	"testing"
+	"time"
 )

+const SeaweedMiniStartupTimeout = 45 * time.Second
+
 func FindBindIP() string {
 	addrs, err := net.InterfaceAddrs()
 	if err != nil {
@@ -54,3 +59,35 @@ func WriteIAMConfig(dir, accessKey, secretKey string) (string, error) {
 	}
 	return iamConfigPath, nil
 }
+
+func MustFreeMiniPort(t *testing.T, name string) int {
+	t.Helper()
+
+	const (
+		minPort = 10000
+		maxPort = 55000
+	)
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	for i := 0; i < 1000; i++ {
+		port := minPort + r.Intn(maxPort-minPort)
+
+		listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+		if err != nil {
+			continue
+		}
+		_ = listener.Close()
+
+		grpcPort := port + 10000
+		grpcListener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
+		if err != nil {
+			continue
+		}
+		_ = grpcListener.Close()
+
+		return port
+	}
+
+	t.Fatalf("failed to get free weed mini port for %s", name)
+	return 0
+}
@@ -0,0 +1,368 @@
+package framework
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+// MixedVolumeCluster wraps a Go master + a mix of Go and Rust volume servers
+// for integration testing. Go servers occupy indices [0, goCount) and Rust
+// servers occupy indices [goCount, goCount+rustCount).
+type MixedVolumeCluster struct {
+	testingTB testing.TB
+	profile   matrix.Profile
+
+	weedBinary       string // Go weed binary (master + Go volume servers)
+	rustVolumeBinary string // Rust volume binary
+
+	baseDir   string
+	configDir string
+	logsDir   string
+	keepLogs  bool
+
+	masterPort     int
+	masterGrpcPort int
+
+	volumePorts     []int
+	volumeGrpcPorts []int
+	volumePubPorts  []int
+	isRust          []bool // which servers are Rust
+
+	masterCmd  *exec.Cmd
+	volumeCmds []*exec.Cmd
+
+	cleanupOnce sync.Once
+}
+
+// StartMixedVolumeCluster starts a cluster with 1 Go master, goCount Go volume
+// servers, and rustCount Rust volume servers. Go servers come first in the index.
+func StartMixedVolumeCluster(t testing.TB, profile matrix.Profile, goCount, rustCount int) *MixedVolumeCluster {
+	t.Helper()
+
+	if goCount < 0 || rustCount < 0 {
+		t.Fatalf("goCount and rustCount must be non-negative, got go=%d rust=%d", goCount, rustCount)
+	}
+	total := goCount + rustCount
+	if total < 2 {
+		t.Fatalf("need at least 2 volume servers, got %d", total)
+	}
+
+	weedBinary, err := FindOrBuildWeedBinary()
+	if err != nil {
+		t.Fatalf("resolve weed binary: %v", err)
+	}
+
+	// Only build the Rust binary when Rust servers are requested.
+	var rustBinary string
+	if rustCount > 0 {
+		rustBinary, err = FindOrBuildRustBinary()
+		if err != nil {
+			t.Skipf("skipping mixed cluster test: rust binary unavailable: %v", err)
+		}
+	}
+
+	baseDir, keepLogs, err := newWorkDir()
+	if err != nil {
+		t.Fatalf("create temp test directory: %v", err)
+	}
+
+	configDir := filepath.Join(baseDir, "config")
+	logsDir := filepath.Join(baseDir, "logs")
+	masterDataDir := filepath.Join(baseDir, "master")
+
+	dirs := []string{configDir, logsDir, masterDataDir}
+	for i := 0; i < total; i++ {
+		dirs = append(dirs, filepath.Join(baseDir, fmt.Sprintf("volume%d", i)))
+	}
+	for _, dir := range dirs {
+		if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
+			t.Fatalf("create %s: %v", dir, mkErr)
+		}
+	}
+
+	if err = writeSecurityConfig(configDir, profile); err != nil {
+		t.Fatalf("write security config: %v", err)
+	}
+
+	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	if err != nil {
+		t.Fatalf("allocate master port pair: %v", err)
+	}
+
+	// 2 ports per server (admin, grpc); add 1 more when public port is split out.
+	portsPerServer := 2
+	if profile.SplitPublicPort {
+		portsPerServer = 3
+	}
+	ports, err := allocatePorts(total * portsPerServer)
+	if err != nil {
+		t.Fatalf("allocate volume ports: %v", err)
+	}
+
+	isRust := make([]bool, total)
+	for i := goCount; i < total; i++ {
+		isRust[i] = true
+	}
+
+	c := &MixedVolumeCluster{
+		testingTB:        t,
+		profile:          profile,
+		weedBinary:       weedBinary,
+		rustVolumeBinary: rustBinary,
+		baseDir:          baseDir,
+		configDir:        configDir,
+		logsDir:          logsDir,
+		keepLogs:         keepLogs,
+		masterPort:       masterPort,
+		masterGrpcPort:   masterGrpcPort,
+		volumePorts:      make([]int, total),
+		volumeGrpcPorts:  make([]int, total),
+		volumePubPorts:   make([]int, total),
+		isRust:           isRust,
+		volumeCmds:       make([]*exec.Cmd, total),
+	}
+
+	for i := 0; i < total; i++ {
+		baseIdx := i * portsPerServer
+		c.volumePorts[i] = ports[baseIdx]
+		c.volumeGrpcPorts[i] = ports[baseIdx+1]
+		if profile.SplitPublicPort {
+			c.volumePubPorts[i] = ports[baseIdx+2]
+		} else {
+			c.volumePubPorts[i] = c.volumePorts[i] // reuse admin port
+		}
+	}
+
+	// Start master
+	if err = c.startMaster(masterDataDir); err != nil {
+		c.Stop()
+		t.Fatalf("start master: %v", err)
+	}
+	helper := &Cluster{logsDir: logsDir}
+	if err = helper.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
+		masterLog := helper.tailLog("master.log")
+		c.Stop()
+		t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
+	}
+
+	// Start volume servers
+	for i := 0; i < total; i++ {
+		volumeDataDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
+		if isRust[i] {
+			err = c.startRustVolume(i, volumeDataDir)
+		} else {
+			err = c.startGoVolume(i, volumeDataDir)
+		}
+		if err != nil {
+			logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
+			c.Stop()
+			t.Fatalf("start volume server %d (rust=%v): %v\nlog tail:\n%s", i, isRust[i], err, logTail)
+		}
+
+		// Rust uses /healthz, Go uses /status
+		healthURL := c.VolumeAdminURL(i) + "/status"
+		if isRust[i] {
+			healthURL = c.VolumeAdminURL(i) + "/healthz"
+		}
+		if err = helper.waitForHTTP(healthURL); err != nil {
+			logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
+			c.Stop()
+			t.Fatalf("wait for volume server %d readiness: %v\nlog tail:\n%s", i, err, logTail)
+		}
+		if err = helper.waitForTCP(c.VolumeGRPCAddress(i)); err != nil {
+			logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
+			c.Stop()
+			t.Fatalf("wait for volume server %d grpc readiness: %v\nlog tail:\n%s", i, err, logTail)
+		}
+	}
+
+	t.Cleanup(func() {
+		c.Stop()
+	})
+
+	return c
+}
+
+func (c *MixedVolumeCluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		for i := len(c.volumeCmds) - 1; i >= 0; i-- {
+			stopProcess(c.volumeCmds[i])
+		}
+		stopProcess(c.masterCmd)
+		if !c.keepLogs && !c.testingTB.Failed() {
+			_ = os.RemoveAll(c.baseDir)
+		} else if c.baseDir != "" {
+			c.testingTB.Logf("mixed volume server integration logs kept at %s", c.baseDir)
+		}
+	})
+}
+
+func (c *MixedVolumeCluster) startMaster(dataDir string) error {
+	logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"master",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.masterPort),
+		"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
+		"-mdir=" + dataDir,
+		"-peers=none",
+		"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
+		"-defaultReplication=000",
+	}
+
+	c.masterCmd = exec.Command(c.weedBinary, args...)
+	c.masterCmd.Dir = c.baseDir
+	c.masterCmd.Stdout = logFile
+	c.masterCmd.Stderr = logFile
+	if err = c.masterCmd.Start(); err != nil {
+		logFile.Close()
+		return err
+	}
+	logFile.Close() // child inherited the fd
+	return nil
+}
+
+func (c *MixedVolumeCluster) startGoVolume(index int, dataDir string) error {
+	logName := fmt.Sprintf("volume%d.log", index)
+	logFile, err := os.Create(filepath.Join(c.logsDir, logName))
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"volume",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.volumePorts[index]),
+		"-port.grpc=" + strconv.Itoa(c.volumeGrpcPorts[index]),
+		"-port.public=" + strconv.Itoa(c.volumePubPorts[index]),
+		"-dir=" + dataDir,
+		"-max=16",
+		"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
+		"-readMode=" + c.profile.ReadMode,
+		"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
+		"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
+	}
+	if c.profile.InflightUploadTimeout > 0 {
+		args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
+	}
+	if c.profile.InflightDownloadTimeout > 0 {
+		args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
+	}
+
+	cmd := exec.Command(c.weedBinary, args...)
+	cmd.Dir = c.baseDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err = cmd.Start(); err != nil {
+		logFile.Close()
+		return err
+	}
+	logFile.Close() // child inherited the fd
+	c.volumeCmds[index] = cmd
+	return nil
+}
+
+func (c *MixedVolumeCluster) startRustVolume(index int, dataDir string) error {
+	logName := fmt.Sprintf("volume%d.log", index)
+	logFile, err := os.Create(filepath.Join(c.logsDir, logName))
+	if err != nil {
+		return err
+	}
+
+	args := rustVolumeArgs(
+		c.profile,
+		c.configDir,
+		c.masterPort,
+		c.volumePorts[index],
+		c.volumeGrpcPorts[index],
+		c.volumePubPorts[index],
+		dataDir,
+	)
+
+	cmd := exec.Command(c.rustVolumeBinary, args...)
+	cmd.Dir = c.baseDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err = cmd.Start(); err != nil {
+		logFile.Close()
+		return err
+	}
+	logFile.Close() // child inherited the fd
+	c.volumeCmds[index] = cmd
+	return nil
+}
+
+// --- accessor methods (mirror MultiVolumeCluster) ---
+
+func (c *MixedVolumeCluster) MasterAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
+}
+
+func (c *MixedVolumeCluster) MasterURL() string {
+	return "http://" + c.MasterAddress()
+}
+
+func (c *MixedVolumeCluster) VolumeAdminAddress(index int) string {
+	if index < 0 || index >= len(c.volumePorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePorts[index]))
+}
+
+func (c *MixedVolumeCluster) VolumePublicAddress(index int) string {
+	if index < 0 || index >= len(c.volumePubPorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPorts[index]))
+}
+
+func (c *MixedVolumeCluster) VolumeGRPCAddress(index int) string {
+	if index < 0 || index >= len(c.volumeGrpcPorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPorts[index]))
+}
+
+func (c *MixedVolumeCluster) VolumeAdminURL(index int) string {
+	return "http://" + c.VolumeAdminAddress(index)
+}
+
+func (c *MixedVolumeCluster) VolumePublicURL(index int) string {
+	return "http://" + c.VolumePublicAddress(index)
+}
+
+func (c *MixedVolumeCluster) BaseDir() string {
+	return c.baseDir
+}
+
+// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort
+func (c *MixedVolumeCluster) VolumeServerAddress(index int) string {
+	if index < 0 || index >= len(c.volumePorts) {
+		return ""
+	}
+	return fmt.Sprintf("%s.%d", c.VolumeAdminAddress(index), c.volumeGrpcPorts[index])
+}
+
+func (c *MixedVolumeCluster) IsRust(index int) bool {
+	if index < 0 || index >= len(c.isRust) {
+		return false
+	}
+	return c.isRust[index]
+}
@@ -0,0 +1,294 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// TestEcDecodePreservesDeletedNeedles verifies that needles deleted via
+// VolumeEcBlobDelete (recorded in .ecj) are correctly excluded from the
+// decoded volume produced by VolumeEcShardsToVolume.
+func TestEcDecodePreservesDeletedNeedles(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const (
+		volumeID = uint32(140)
+		keyA     = uint64(990020)
+		cookieA  = uint32(0xDA001122)
+		keyB     = uint64(990021)
+		cookieB  = uint32(0xDA003344)
+	)
+
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fidA := framework.NewFileID(volumeID, keyA, cookieA)
+	fidB := framework.NewFileID(volumeID, keyB, cookieB)
+	payloadA := []byte("needle-A-should-be-deleted-after-decode")
+	payloadB := []byte("needle-B-should-survive-decode")
+
+	// Upload two needles.
+	for _, tc := range []struct {
+		fid     string
+		payload []byte
+	}{
+		{fidA, payloadA},
+		{fidB, payloadB},
+	} {
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, tc.payload)
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s: expected 201, got %d", tc.fid, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// EC encode.
+	_, err := client.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate: %v", err)
+	}
+
+	// Mount all data shards so the EC volume is usable.
+	_, err = client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID, Collection: "",
+		ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount: %v", err)
+	}
+
+	// Delete needle A via EC path (writes to .ecj).
+	_, err = client.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId: volumeID, Collection: "",
+		FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete needle A: %v", err)
+	}
+
+	// Unmount the normal volume so decode writes fresh files.
+	_, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeUnmount: %v", err)
+	}
+
+	// Decode EC shards back to a normal volume.
+	_, err = client.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsToVolume: %v", err)
+	}
+
+	// Re-mount the decoded volume.
+	_, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeMount: %v", err)
+	}
+
+	// Needle A should be gone (deleted via .ecj before decode).
+	respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA)
+	bodyA := framework.ReadAllAndClose(t, respA)
+	if respA.StatusCode >= 500 {
+		t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
+	}
+	if respA.StatusCode != http.StatusNotFound {
+		t.Fatalf("needle A should be 404 after decode, got %d", respA.StatusCode)
+	}
+
+	// Needle B should still be readable.
+	respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB)
+	bodyB := framework.ReadAllAndClose(t, respB)
+	if respB.StatusCode != http.StatusOK {
+		t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
+	}
+	if string(bodyB) != string(payloadB) {
+		t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
+	}
+}
+
+// TestEcDecodeCollectsEcjFromPeer verifies that .ecj deletion entries from a
+// peer server that contributes no new data shards are still collected during
+// decode. This is the regression test for the fix in collectEcShards that
+// always copies .ecj from every shard location.
+//
+// Scenario:
+//   - Server 0 holds all 10 data shards (decode target).
+//   - Server 1 holds a copy of shard 0 (no new shards for server 0).
+//   - A needle is deleted ONLY on server 1 (server 0's .ecj is empty).
+//   - During decode on server 0, server 1's .ecj must be collected and applied.
+func TestEcDecodeCollectsEcjFromPeer(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMultiVolumeClusterAuto(t, matrix.P1(), 2)
+	conn0, client0 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, client1 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	const (
+		volumeID = uint32(141)
+		keyA     = uint64(990030)
+		cookieA  = uint32(0xDB001122)
+		keyB     = uint64(990031)
+		cookieB  = uint32(0xDB003344)
+	)
+
+	// Allocate and upload on server 0.
+	framework.AllocateVolume(t, client0, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fidA := framework.NewFileID(volumeID, keyA, cookieA)
+	fidB := framework.NewFileID(volumeID, keyB, cookieB)
+	payloadB := []byte("needle-B-should-survive-peer-ecj-decode")
+
+	resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA, []byte("needle-A-deleted-on-peer"))
+	_ = framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload A: expected 201, got %d", resp.StatusCode)
+	}
+	resp = framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB, payloadB)
+	_ = framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload B: expected 201, got %d", resp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// EC encode on server 0.
+	_, err := client0.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate on server 0: %v", err)
+	}
+
+	// Build the SourceDataNode address for server 0 (format: host:adminPort.grpcPort).
+	sourceDataNode := cluster.VolumeAdminAddress(0) + "." +
+		strings.Split(cluster.VolumeGRPCAddress(0), ":")[1]
+
+	// Copy shard 0 + ecx + ecj from server 0 → server 1.
+	_, err = client1.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: sourceDataNode,
+		ShardIds:       []uint32{0},
+		CopyEcxFile:    true,
+		CopyEcjFile:    true,
+		CopyVifFile:    true,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsCopy 0→1: %v", err)
+	}
+
+	// Mount shard 0 on server 1 so the EC volume can accept deletions.
+	_, err = client1.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID, Collection: "",
+		ShardIds: []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount on server 1: %v", err)
+	}
+
+	// Delete needle A on server 1 only (creates .ecj entry on server 1).
+	_, err = client1.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId: volumeID, Collection: "",
+		FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete needle A on server 1: %v", err)
+	}
+
+	// Mount all data shards on server 0 (the decode target).
+	_, err = client0.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID, Collection: "",
+		ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount on server 0: %v", err)
+	}
+
+	// Collect .ecj from server 1 → server 0 with NO new shard IDs.
+	// This is the critical path: server 1 has shard 0 which server 0 already
+	// has, so needToCopyShardsInfo would be empty. Before the fix in
+	// collectEcShards, this copy would be skipped entirely, losing server 1's
+	// deletion entries.
+	server1DataNode := cluster.VolumeAdminAddress(1) + "." +
+		strings.Split(cluster.VolumeGRPCAddress(1), ":")[1]
+
+	_, err = client0.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: server1DataNode,
+		ShardIds:       []uint32{}, // No new shards — just .ecj.
+		CopyEcxFile:    false,
+		CopyEcjFile:    true,
+		CopyVifFile:    false,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsCopy .ecj from server 1→0: %v", err)
+	}
+
+	// Unmount the normal volume before decode.
+	_, err = client0.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeUnmount on server 0: %v", err)
+	}
+
+	// Decode on server 0. RebuildEcxFile should see needle A's deletion from
+	// the .ecj that was collected from server 1.
+	_, err = client0.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsToVolume on server 0: %v", err)
+	}
+
+	// Re-mount the decoded normal volume.
+	_, err = client0.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeMount on server 0: %v", err)
+	}
+
+	// Needle A should be gone — its deletion was in server 1's .ecj.
+	respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA)
+	bodyA := framework.ReadAllAndClose(t, respA)
+	if respA.StatusCode >= 500 {
+		t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
+	}
+	if respA.StatusCode != http.StatusNotFound {
+		t.Fatalf("needle A should be 404 (ecj from peer), got %d", respA.StatusCode)
+	}
+
+	// Needle B should still be readable.
+	respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB)
+	bodyB := framework.ReadAllAndClose(t, respB)
+	if respB.StatusCode != http.StatusOK {
+		t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
+	}
+	if string(bodyB) != string(payloadB) {
+		t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
+	}
+}
@@ -0,0 +1,567 @@
+package volume_server_grpc_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+// deleteAndWaitForTombstone issues an HTTP DELETE for the given fid on
+// volumeURL, asserts a successful response, then polls with GET until
+// the file returns 404 (tombstone visible) or the timeout elapses.
+func deleteAndWaitForTombstone(t *testing.T, httpClient *http.Client, volumeURL, fid string) {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/%s", volumeURL, fid), nil)
+	if err != nil {
+		t.Fatalf("build delete request for %s: %v", fid, err)
+	}
+	resp := framework.DoRequest(t, httpClient, req)
+	framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK {
+		t.Fatalf("delete %s: expected 200 or 202, got %d", fid, resp.StatusCode)
+	}
+	// Poll until GET returns 404 (tombstone flushed to disk).
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		getResp := framework.ReadBytes(t, httpClient, volumeURL, fid)
+		status := getResp.StatusCode
+		framework.ReadAllAndClose(t, getResp)
+		if status == http.StatusNotFound {
+			return
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Fatalf("delete %s: tombstone not visible after 5s", fid)
+}
+
+// TestMixedBalanceCopyGoToRust verifies that VolumeCopy works from a Go volume
+// server to a Rust volume server. This is the core operation behind volume
+// balancing in a mixed Go+Rust cluster.
+func TestMixedBalanceCopyGoToRust(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
+
+	// server 0 = Go, server 1 = Rust
+	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	httpClient := framework.NewHTTPClient()
+
+	const volumeID = uint32(50)
+
+	// Allocate volume on Go server and upload test data
+	framework.AllocateVolume(t, goClient, volumeID, "")
+
+	testFiles := []struct {
+		key    uint64
+		cookie uint32
+		data   []byte
+	}{
+		{1, 0xAABBCCDD, []byte("hello from Go server")},
+		{2, 0x11223344, []byte("second file for balance test")},
+		{3, 0xDEADBEEF, make([]byte, 4096)}, // larger file
+	}
+
+	for _, f := range testFiles {
+		fid := framework.NewFileID(volumeID, f.key, f.cookie)
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, f.data)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// Read source volume status before copy
+	sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
+	}
+	t.Logf("Source: dat=%d idx=%d files=%d version=%d",
+		sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
+		sourceStatus.GetFileCount(), sourceStatus.GetVersion())
+
+	// Copy volume from Go (server 0) to Rust (server 1)
+	copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy call failed: %v", err)
+	}
+
+	var lastAppendAtNs uint64
+	for {
+		resp, recvErr := copyStream.Recv()
+		if recvErr != nil {
+			if recvErr != io.EOF {
+				t.Fatalf("VolumeCopy recv error: %v", recvErr)
+			}
+			break
+		}
+		if resp.GetLastAppendAtNs() != 0 {
+			lastAppendAtNs = resp.GetLastAppendAtNs()
+		}
+	}
+	t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
+
+	// Verify: read volume status from Rust server
+	targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
+	}
+	t.Logf("Target: dat=%d idx=%d files=%d version=%d",
+		targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
+		targetStatus.GetFileCount(), targetStatus.GetVersion())
+
+	if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
+		t.Fatalf("dat file size mismatch: source=%d target=%d",
+			sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
+	}
+	if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
+		t.Fatalf("idx file size mismatch: source=%d target=%d",
+			sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
+	}
+	if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
+		t.Fatalf("file count mismatch: source=%d target=%d",
+			sourceStatus.GetFileCount(), targetStatus.GetFileCount())
+	}
+
+	// Verify data can be read from Rust server
+	for _, f := range testFiles {
+		fid := framework.NewFileID(volumeID, f.key, f.cookie)
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("read %s from Rust server: expected 200, got %d", fid, resp.StatusCode)
+		}
+		if !bytes.Equal(body, f.data) {
+			t.Fatalf("read %s from Rust server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
+		}
+	}
+}
+
+// TestMixedBalanceCopyRustToGo verifies that VolumeCopy works from a Rust
+// volume server to a Go volume server.
+func TestMixedBalanceCopyRustToGo(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
+
+	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	httpClient := framework.NewHTTPClient()
+
+	const volumeID = uint32(51)
+
+	// Allocate volume on Rust server and upload test data
+	framework.AllocateVolume(t, rustClient, volumeID, "")
+
+	testFiles := []struct {
+		key    uint64
+		cookie uint32
+		data   []byte
+	}{
+		{1, 0xAABBCCDD, []byte("hello from Rust server")},
+		{2, 0x11223344, []byte("second file for reverse balance")},
+	}
+
+	for _, f := range testFiles {
+		fid := framework.NewFileID(volumeID, f.key, f.cookie)
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid, f.data)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	sourceStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
+	}
+
+	// Copy volume from Rust (server 1) to Go (server 0)
+	copyStream, err := goClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: cluster.VolumeServerAddress(1),
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy call failed: %v", err)
+	}
+	for {
+		_, recvErr := copyStream.Recv()
+		if recvErr != nil {
+			if recvErr != io.EOF {
+				t.Fatalf("VolumeCopy recv error: %v", recvErr)
+			}
+			break
+		}
+	}
+
+	targetStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
+	}
+
+	if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
+		t.Fatalf("dat file size mismatch: source=%d target=%d",
+			sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
+	}
+	if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
+		t.Fatalf("idx file size mismatch: source=%d target=%d",
+			sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
+	}
+	if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
+		t.Fatalf("file count mismatch: source=%d target=%d",
+			sourceStatus.GetFileCount(), targetStatus.GetFileCount())
+	}
+
+	// Verify data can be read from Go server
+	for _, f := range testFiles {
+		fid := framework.NewFileID(volumeID, f.key, f.cookie)
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("read %s from Go server: expected 200, got %d", fid, resp.StatusCode)
+		}
+		if !bytes.Equal(body, f.data) {
+			t.Fatalf("read %s from Go server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
+		}
+	}
+}
+
+// TestMixedBalanceCopyWithDeletes verifies that VolumeCopy correctly handles
+// volumes that have both active and deleted needles.
+func TestMixedBalanceCopyWithDeletes(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
+
+	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	httpClient := framework.NewHTTPClient()
+
+	const volumeID = uint32(52)
+	framework.AllocateVolume(t, goClient, volumeID, "")
+
+	// Upload files
+	for i := uint64(1); i <= 5; i++ {
+		fid := framework.NewFileID(volumeID, i, 0x12340000+uint32(i))
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("file-%d", i)))
+		framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
+		}
+	}
+
+	// Delete some files and wait for tombstones to be visible
+	for _, key := range []uint64{2, 4} {
+		fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
+		deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0), fid)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus: %v", err)
+	}
+	t.Logf("Source after deletes: dat=%d idx=%d files=%d",
+		sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
+		sourceStatus.GetFileCount())
+
+	// Copy volume from Go to Rust
+	copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy: %v", err)
+	}
+	var lastAppendAtNs uint64
+	for {
+		resp, recvErr := copyStream.Recv()
+		if recvErr != nil {
+			if recvErr != io.EOF {
+				t.Fatalf("VolumeCopy recv error: %v", recvErr)
+			}
+			break
+		}
+		if resp.GetLastAppendAtNs() != 0 {
+			lastAppendAtNs = resp.GetLastAppendAtNs()
+		}
+	}
+	if lastAppendAtNs == 0 {
+		t.Fatalf("VolumeCopy did not return a lastAppendAtNs timestamp")
+	}
+	t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
+
+	targetStatusAfterCopy, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Rust: %v", err)
+	}
+	t.Logf("Target after copy: dat=%d idx=%d files=%d",
+		targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterCopy.GetIdxFileSize(),
+		targetStatusAfterCopy.GetFileCount())
+
+	if sourceStatus.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
+		t.Fatalf("dat file size mismatch: source=%d target=%d",
+			sourceStatus.GetDatFileSize(), targetStatusAfterCopy.GetDatFileSize())
+	}
+	if sourceStatus.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
+		t.Fatalf("idx file size mismatch: source=%d target=%d",
+			sourceStatus.GetIdxFileSize(), targetStatusAfterCopy.GetIdxFileSize())
+	}
+
+	// Tail from the copy checkpoint — source is unchanged (deletes happened
+	// before copy), so tailing should not append any data.
+	_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
+		VolumeId:           volumeID,
+		SinceNs:            lastAppendAtNs,
+		IdleTimeoutSeconds: 3,
+		SourceVolumeServer: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeTailReceiver: %v", err)
+	}
+
+	targetStatusAfterTail, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus after tail: %v", err)
+	}
+	if targetStatusAfterTail.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
+		t.Fatalf("dat grew after tail: before=%d after=%d",
+			targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterTail.GetDatFileSize())
+	}
+	if targetStatusAfterTail.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
+		t.Fatalf("idx grew after tail: before=%d after=%d",
+			targetStatusAfterCopy.GetIdxFileSize(), targetStatusAfterTail.GetIdxFileSize())
+	}
+
+	// Verify surviving files are readable from Rust
+	for _, key := range []uint64{1, 3, 5} {
+		fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("read surviving file %d from Rust: expected 200, got %d", key, resp.StatusCode)
+		}
+		expected := fmt.Sprintf("file-%d", key)
+		if string(body) != expected {
+			t.Fatalf("data mismatch for file %d: got %q want %q", key, body, expected)
+		}
+	}
+
+	// Verify deleted files return 404
+	for _, key := range []uint64{2, 4} {
+		fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusNotFound {
+			t.Fatalf("read deleted file %d from Rust: expected 404, got %d", key, resp.StatusCode)
+		}
+	}
+}
+
+// TestMixedBalanceFullMoveGoToRust exercises the complete volume balance move
+// flow: mark readonly → copy → tail → verify sizes → delete source.
+// This mirrors the steps in balance_task.go Execute().
+func TestMixedBalanceFullMoveGoToRust(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
+
+	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	httpClient := framework.NewHTTPClient()
+
+	const volumeID = uint32(53)
+	framework.AllocateVolume(t, goClient, volumeID, "")
+
+	// Upload test data
+	for i := uint64(1); i <= 5; i++ {
+		fid := framework.NewFileID(volumeID, i, 0xABCD0000+uint32(i))
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("balance-move-file-%d", i)))
+		framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// Step 1: Copy volume to Rust server (source stays writable so we can
+	// delete after copy to exercise the tail tombstone path)
+	copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy: %v", err)
+	}
+	var lastAppendAtNs uint64
+	for {
+		resp, recvErr := copyStream.Recv()
+		if recvErr != nil {
+			if recvErr != io.EOF {
+				t.Fatalf("VolumeCopy recv: %v", recvErr)
+			}
+			break
+		}
+		if resp.GetLastAppendAtNs() != 0 {
+			lastAppendAtNs = resp.GetLastAppendAtNs()
+		}
+	}
+	t.Logf("Copy done, lastAppendAtNs=%d", lastAppendAtNs)
+
+	// Step 2: Delete file 3 on the source AFTER copy. This creates a
+	// tombstone needle that the tail step must propagate to the Rust server.
+	deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0),
+		framework.NewFileID(volumeID, 3, 0xABCD0003))
+
+	// Step 3: Mark source readonly so no further writes arrive during tail
+	_, err = goClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("mark readonly: %v", err)
+	}
+
+	// Read source status (the reference for post-tail verification)
+	sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("read source status: %v", err)
+	}
+	t.Logf("Source: dat=%d idx=%d files=%d",
+		sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
+		sourceStatus.GetFileCount())
+
+	// Step 4: Tail for updates — this must pick up the delete tombstone
+	_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
+		VolumeId:           volumeID,
+		SinceNs:            lastAppendAtNs,
+		IdleTimeoutSeconds: 5,
+		SourceVolumeServer: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeTailReceiver: %v", err)
+	}
+
+	// Step 5: Verify dat/idx sizes match after tail.
+	// We compare file sizes (byte-level correctness) rather than file_count
+	// because the tail writes the tombstone via the write path (which
+	// increments file_count) while the source used the delete path (which
+	// only increments deletion_count). This is consistent with Go behavior.
+	targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("read target status: %v", err)
+	}
+	t.Logf("Target: dat=%d idx=%d files=%d",
+		targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
+		targetStatus.GetFileCount())
+
+	if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
+		t.Fatalf("dat size mismatch after tail: source=%d target=%d",
+			sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
+	}
+	if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
+		t.Fatalf("idx size mismatch after tail: source=%d target=%d",
+			sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
+	}
+
+	// Step 6: Delete volume from source
+	_, err = goClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("delete source volume: %v", err)
+	}
+
+	// Verify source volume is gone
+	_, err = goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err == nil {
+		t.Fatalf("expected error reading deleted source volume, got nil")
+	}
+	t.Logf("Source volume deleted successfully (error as expected: %v)", err)
+
+	// Verify all surviving data is readable from Rust (the new home)
+	for _, key := range []uint64{1, 2, 4, 5} {
+		fid := framework.NewFileID(volumeID, key, 0xABCD0000+uint32(key))
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("read file %d from Rust after move: expected 200, got %d", key, resp.StatusCode)
+		}
+		expected := fmt.Sprintf("balance-move-file-%d", key)
+		if string(body) != expected {
+			t.Fatalf("data mismatch for file %d after move: got %q want %q", key, body, expected)
+		}
+	}
+
+	// Verify deleted file is 404 on Rust (tombstone propagated via tail)
+	{
+		fid := framework.NewFileID(volumeID, 3, 0xABCD0003)
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusNotFound {
+			t.Fatalf("deleted file 3 on Rust: expected 404, got %d", resp.StatusCode)
+		}
+	}
+
+	t.Logf("Full balance move completed: volume %d moved from Go to Rust, source purged", volumeID)
+}
@@ -45,6 +45,7 @@ type ObjectStoreUser struct {
 	SecretKey   string   `json:"secret_key"`
 	Permissions []string `json:"permissions"`
 	PolicyNames []string `json:"policy_names"`
+	IsStatic    bool     `json:"is_static"` // loaded from static config file, not editable
 }

 type ObjectStoreUsersData struct {
@@ -867,6 +867,24 @@ func (s *AdminServer) DeleteS3Bucket(bucketName string) error {
 	})
 }

+// IsStaticUser checks if a user is a static identity by loading the
+// configuration from the credential manager and checking the IsStatic flag.
+func (s *AdminServer) IsStaticUser(username string) bool {
+	if s.credentialManager == nil {
+		return false
+	}
+	s3cfg, err := s.credentialManager.LoadConfiguration(context.Background())
+	if err != nil {
+		return false
+	}
+	for _, ident := range s3cfg.Identities {
+		if ident.Name == username {
+			return ident.IsStatic
+		}
+	}
+	return false
+}
+
 // GetObjectStoreUsers retrieves object store users from identity.json
 func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUser, error) {
 	if s.credentialManager == nil {
@@ -890,6 +908,7 @@ func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUse
 		user := ObjectStoreUser{
 			Username:    identity.Name,
 			Permissions: identity.Actions,
+			IsStatic:    identity.IsStatic,
 		}

 		// Set email from account if available
@@ -943,7 +962,7 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 			leaderCount++
 		}

-		masterMap[master.Address] = masterInfo
+		masterMap[masterInfo.Address] = masterInfo
 	}

 	// Then, get additional master information from Raft cluster
@@ -955,11 +974,11 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {

 		// Process each raft server
 		for _, server := range resp.ClusterServers {
-			address := server.Address
-			httpAddress := pb.ServerAddress(address).ToHttpAddress()
+			// Raft stores gRPC addresses, convert to HTTP address
+			httpAddress := pb.GrpcAddressToServerAddress(server.Address)

 			// Update existing master info or create new one
-			if masterInfo, exists := masterMap[address]; exists {
+			if masterInfo, exists := masterMap[httpAddress]; exists {
 				// Update existing master with raft data
 				masterInfo.IsLeader = server.IsLeader
 				masterInfo.Suffrage = server.Suffrage
@@ -970,7 +989,7 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 					IsLeader: server.IsLeader,
 					Suffrage: server.Suffrage,
 				}
-				masterMap[address] = masterInfo
+				masterMap[httpAddress] = masterInfo
 			}

 			if server.IsLeader {
@@ -1646,159 +1665,6 @@ func (as *AdminServer) GetConfigPersistence() *ConfigPersistence {
 	return as.configPersistence
 }

-// convertJSONToMaintenanceConfig converts JSON map to protobuf MaintenanceConfig
-func convertJSONToMaintenanceConfig(jsonConfig map[string]interface{}) (*maintenance.MaintenanceConfig, error) {
-	config := &maintenance.MaintenanceConfig{}
-
-	// Helper function to get int32 from interface{}
-	getInt32 := func(key string) (int32, error) {
-		if val, ok := jsonConfig[key]; ok {
-			switch v := val.(type) {
-			case int:
-				return int32(v), nil
-			case int32:
-				return v, nil
-			case int64:
-				return int32(v), nil
-			case float64:
-				return int32(v), nil
-			default:
-				return 0, fmt.Errorf("invalid type for %s: expected number, got %T", key, v)
-			}
-		}
-		return 0, nil
-	}
-
-	// Helper function to get bool from interface{}
-	getBool := func(key string) bool {
-		if val, ok := jsonConfig[key]; ok {
-			if b, ok := val.(bool); ok {
-				return b
-			}
-		}
-		return false
-	}
-
-	var err error
-
-	// Convert basic fields
-	config.Enabled = getBool("enabled")
-
-	if config.ScanIntervalSeconds, err = getInt32("scan_interval_seconds"); err != nil {
-		return nil, err
-	}
-	if config.WorkerTimeoutSeconds, err = getInt32("worker_timeout_seconds"); err != nil {
-		return nil, err
-	}
-	if config.TaskTimeoutSeconds, err = getInt32("task_timeout_seconds"); err != nil {
-		return nil, err
-	}
-	if config.RetryDelaySeconds, err = getInt32("retry_delay_seconds"); err != nil {
-		return nil, err
-	}
-	if config.MaxRetries, err = getInt32("max_retries"); err != nil {
-		return nil, err
-	}
-	if config.CleanupIntervalSeconds, err = getInt32("cleanup_interval_seconds"); err != nil {
-		return nil, err
-	}
-	if config.TaskRetentionSeconds, err = getInt32("task_retention_seconds"); err != nil {
-		return nil, err
-	}
-
-	// Convert policy if present
-	if policyData, ok := jsonConfig["policy"]; ok {
-		if policyMap, ok := policyData.(map[string]interface{}); ok {
-			policy := &maintenance.MaintenancePolicy{}
-
-			if globalMaxConcurrent, err := getInt32FromMap(policyMap, "global_max_concurrent"); err != nil {
-				return nil, err
-			} else {
-				policy.GlobalMaxConcurrent = globalMaxConcurrent
-			}
-
-			if defaultRepeatIntervalSeconds, err := getInt32FromMap(policyMap, "default_repeat_interval_seconds"); err != nil {
-				return nil, err
-			} else {
-				policy.DefaultRepeatIntervalSeconds = defaultRepeatIntervalSeconds
-			}
-
-			if defaultCheckIntervalSeconds, err := getInt32FromMap(policyMap, "default_check_interval_seconds"); err != nil {
-				return nil, err
-			} else {
-				policy.DefaultCheckIntervalSeconds = defaultCheckIntervalSeconds
-			}
-
-			// Convert task policies if present
-			if taskPoliciesData, ok := policyMap["task_policies"]; ok {
-				if taskPoliciesMap, ok := taskPoliciesData.(map[string]interface{}); ok {
-					policy.TaskPolicies = make(map[string]*maintenance.TaskPolicy)
-
-					for taskType, taskPolicyData := range taskPoliciesMap {
-						if taskPolicyMap, ok := taskPolicyData.(map[string]interface{}); ok {
-							taskPolicy := &maintenance.TaskPolicy{}
-
-							taskPolicy.Enabled = getBoolFromMap(taskPolicyMap, "enabled")
-
-							if maxConcurrent, err := getInt32FromMap(taskPolicyMap, "max_concurrent"); err != nil {
-								return nil, err
-							} else {
-								taskPolicy.MaxConcurrent = maxConcurrent
-							}
-
-							if repeatIntervalSeconds, err := getInt32FromMap(taskPolicyMap, "repeat_interval_seconds"); err != nil {
-								return nil, err
-							} else {
-								taskPolicy.RepeatIntervalSeconds = repeatIntervalSeconds
-							}
-
-							if checkIntervalSeconds, err := getInt32FromMap(taskPolicyMap, "check_interval_seconds"); err != nil {
-								return nil, err
-							} else {
-								taskPolicy.CheckIntervalSeconds = checkIntervalSeconds
-							}
-
-							policy.TaskPolicies[taskType] = taskPolicy
-						}
-					}
-				}
-			}
-
-			config.Policy = policy
-		}
-	}
-
-	return config, nil
-}
-
-// Helper functions for map conversion
-func getInt32FromMap(m map[string]interface{}, key string) (int32, error) {
-	if val, ok := m[key]; ok {
-		switch v := val.(type) {
-		case int:
-			return int32(v), nil
-		case int32:
-			return v, nil
-		case int64:
-			return int32(v), nil
-		case float64:
-			return int32(v), nil
-		default:
-			return 0, fmt.Errorf("invalid type for %s: expected number, got %T", key, v)
-		}
-	}
-	return 0, nil
-}
-
-func getBoolFromMap(m map[string]interface{}, key string) bool {
-	if val, ok := m[key]; ok {
-		if b, ok := val.(bool); ok {
-			return b
-		}
-	}
-	return false
-}
-
 type collectionStats struct {
 	PhysicalSize int64
 	LogicalSize  int64
@@ -361,26 +361,6 @@ func normalizeQuotaUnit(unit string) (string, error) {
 	}
 }

-// Helper function to convert bytes to appropriate unit and size
-func convertBytesToQuota(bytes int64) (int64, string) {
-	if bytes == 0 {
-		return 0, "MB"
-	}
-
-	// Convert to TB if >= 1TB
-	if bytes >= 1024*1024*1024*1024 && bytes%(1024*1024*1024*1024) == 0 {
-		return bytes / (1024 * 1024 * 1024 * 1024), "TB"
-	}
-
-	// Convert to GB if >= 1GB
-	if bytes >= 1024*1024*1024 && bytes%(1024*1024*1024) == 0 {
-		return bytes / (1024 * 1024 * 1024), "GB"
-	}
-
-	// Convert to MB (default)
-	return bytes / (1024 * 1024), "MB"
-}
-
 // SetBucketQuota sets the quota for a bucket
 func (s *AdminServer) SetBucketQuota(bucketName string, quotaBytes int64, quotaEnabled bool) error {
 	return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
@@ -506,18 +506,6 @@ func getShardCount(ecIndexBits uint32) int {
 	return count
 }

-// getMissingShards returns a slice of missing shard IDs for a volume
-// Assumes default 10+4 EC configuration (14 total shards)
-func getMissingShards(ecIndexBits uint32) []int {
-	var missing []int
-	for i := 0; i < erasure_coding.TotalShardsCount; i++ {
-		if (ecIndexBits & (1 << uint(i))) == 0 {
-			missing = append(missing, i)
-		}
-	}
-	return missing
-}
-
 // sortEcShards sorts EC shards based on the specified field and order
 func sortEcShards(shards []EcShardWithInfo, sortBy string, sortOrder string) {
 	sort.Slice(shards, func(i, j int) bool {
@@ -430,67 +430,6 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
 	return offsets, nil
 }

-// convertRecordTypeToSchemaFields converts a protobuf RecordType to SchemaFieldInfo slice
-func convertRecordTypeToSchemaFields(recordType *schema_pb.RecordType) []SchemaFieldInfo {
-	var schemaFields []SchemaFieldInfo
-
-	if recordType == nil || recordType.Fields == nil {
-		return schemaFields
-	}
-
-	for _, field := range recordType.Fields {
-		schemaField := SchemaFieldInfo{
-			Name:     field.Name,
-			Type:     getFieldTypeString(field.Type),
-			Required: field.IsRequired,
-		}
-		schemaFields = append(schemaFields, schemaField)
-	}
-
-	return schemaFields
-}
-
-// getFieldTypeString converts a protobuf Type to a human-readable string
-func getFieldTypeString(fieldType *schema_pb.Type) string {
-	if fieldType == nil {
-		return "unknown"
-	}
-
-	switch kind := fieldType.Kind.(type) {
-	case *schema_pb.Type_ScalarType:
-		return getScalarTypeString(kind.ScalarType)
-	case *schema_pb.Type_RecordType:
-		return "record"
-	case *schema_pb.Type_ListType:
-		elementType := getFieldTypeString(kind.ListType.ElementType)
-		return fmt.Sprintf("list<%s>", elementType)
-	default:
-		return "unknown"
-	}
-}
-
-// getScalarTypeString converts a protobuf ScalarType to a string
-func getScalarTypeString(scalarType schema_pb.ScalarType) string {
-	switch scalarType {
-	case schema_pb.ScalarType_BOOL:
-		return "bool"
-	case schema_pb.ScalarType_INT32:
-		return "int32"
-	case schema_pb.ScalarType_INT64:
-		return "int64"
-	case schema_pb.ScalarType_FLOAT:
-		return "float"
-	case schema_pb.ScalarType_DOUBLE:
-		return "double"
-	case schema_pb.ScalarType_BYTES:
-		return "bytes"
-	case schema_pb.ScalarType_STRING:
-		return "string"
-	default:
-		return "unknown"
-	}
-}
-
 // convertTopicPublishers converts protobuf TopicPublisher slice to PublisherInfo slice
 func convertTopicPublishers(publishers []*mq_pb.TopicPublisher) []PublisherInfo {
 	publisherInfos := make([]PublisherInfo, 0, len(publishers))
@@ -2,8 +2,6 @@ package dash

 import (
 	"context"
-	"crypto/rand"
-	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -851,43 +849,6 @@ func normalizeTimeout(timeoutSeconds int, defaultTimeout, maxTimeout time.Durati
 	return timeout
 }

-func buildJobSpecFromProposal(jobType string, proposal *plugin_pb.JobProposal, index int) *plugin_pb.JobSpec {
-	now := timestamppb.Now()
-	suffix := make([]byte, 4)
-	if _, err := rand.Read(suffix); err != nil {
-		// Fallback to simpler ID if rand fails
-		suffix = []byte(fmt.Sprintf("%d", index))
-	}
-	jobID := fmt.Sprintf("%s-%d-%s", jobType, now.AsTime().UnixNano(), hex.EncodeToString(suffix))
-
-	jobSpec := &plugin_pb.JobSpec{
-		JobId:      jobID,
-		JobType:    jobType,
-		Priority:   plugin_pb.JobPriority_JOB_PRIORITY_NORMAL,
-		CreatedAt:  now,
-		Labels:     make(map[string]string),
-		Parameters: make(map[string]*plugin_pb.ConfigValue),
-		DedupeKey:  "",
-	}
-
-	if proposal != nil {
-		jobSpec.Summary = proposal.Summary
-		jobSpec.Detail = proposal.Detail
-		if proposal.Priority != plugin_pb.JobPriority_JOB_PRIORITY_UNSPECIFIED {
-			jobSpec.Priority = proposal.Priority
-		}
-		jobSpec.DedupeKey = proposal.DedupeKey
-		jobSpec.Parameters = plugin.CloneConfigValueMap(proposal.Parameters)
-		if proposal.Labels != nil {
-			for k, v := range proposal.Labels {
-				jobSpec.Labels[k] = v
-			}
-		}
-	}
-
-	return jobSpec
-}
-
 func applyDescriptorDefaultsToPersistedConfig(
 	config *plugin_pb.PersistedJobTypeConfig,
 	descriptor *plugin_pb.JobTypeDescriptor,
@@ -115,32 +115,6 @@ func TestExpirePluginJobAPI(t *testing.T) {
 	})
 }

-func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
-	t.Parallel()
-
-	proposal := &plugin_pb.JobProposal{
-		ProposalId: "vacuum-2",
-		DedupeKey:  "vacuum:2",
-		JobType:    "vacuum",
-	}
-
-	jobA := buildJobSpecFromProposal("vacuum", proposal, 0)
-	jobB := buildJobSpecFromProposal("vacuum", proposal, 1)
-
-	if jobA.JobId == proposal.ProposalId {
-		t.Fatalf("job id must not reuse proposal id: %s", jobA.JobId)
-	}
-	if jobB.JobId == proposal.ProposalId {
-		t.Fatalf("job id must not reuse proposal id: %s", jobB.JobId)
-	}
-	if jobA.JobId == jobB.JobId {
-		t.Fatalf("job ids must be unique across jobs: %s", jobA.JobId)
-	}
-	if jobA.DedupeKey != proposal.DedupeKey {
-		t.Fatalf("dedupe key must be preserved: got=%s want=%s", jobA.DedupeKey, proposal.DedupeKey)
-	}
-}
-
 func TestApplyDescriptorDefaultsToPersistedConfigBackfillsAdminDefaults(t *testing.T) {
 	t.Parallel()

@@ -447,11 +447,6 @@ type QueueStats = maintenance.QueueStats
 type WorkerDetailsData = maintenance.WorkerDetailsData
 type WorkerPerformance = maintenance.WorkerPerformance

-// GetTaskIcon returns the icon CSS class for a task type from its UI provider
-func GetTaskIcon(taskType MaintenanceTaskType) string {
-	return maintenance.GetTaskIcon(taskType)
-}
-
 // Status constants (these are still static)
 const (
 	TaskStatusPending    = maintenance.TaskStatusPending
@@ -3,13 +3,13 @@ package dash
 import (
 	"context"
 	"crypto/rand"
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"strings"
 	"time"

 	"github.com/seaweedfs/seaweedfs/weed/credential"
+	"github.com/seaweedfs/seaweedfs/weed/iam"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 )

@@ -175,7 +175,7 @@ func (s *AdminServer) GetObjectStoreUserDetails(username string) (*UserDetails,

 	ctx := context.Background()

-	// Get user using credential manager
+	// Get user using credential manager (resolves static users via filer gRPC)
 	identity, err := s.credentialManager.GetUser(ctx, username)
 	if err != nil {
 		if err == credential.ErrUserNotFound {
@@ -435,10 +435,13 @@ func generateAccessKey() string {
 }

 func generateSecretKey() string {
-	// Generate 40-character secret key (AWS standard)
-	b := make([]byte, 30) // 30 bytes = 40 characters in base64
-	rand.Read(b)
-	return base64.StdEncoding.EncodeToString(b)
+	// Use the IAM helper to generate URL-safe secret keys (no +, / characters)
+	// that won't break S3 signature authentication
+	key, err := iam.GenerateSecretAccessKey()
+	if err != nil {
+		panic(fmt.Sprintf("failed to generate secret key: %v", err))
+	}
+	return key
 }

 func generateAccountId() string {
@@ -74,9 +74,16 @@ func TestGenerateSecretKey(t *testing.T) {
 	key1 := generateSecretKey()
 	key2 := generateSecretKey()

-	// Check length (base64 encoding of 30 bytes = 40 characters)
-	if len(key1) != 40 {
-		t.Errorf("Expected secret key length 40, got %d", len(key1))
+	// Check length (IAM standard secret key length)
+	if len(key1) != 42 {
+		t.Errorf("Expected secret key length 42, got %d", len(key1))
+	}
+
+	// Check that key contains only URL-safe characters (no +, /)
+	for _, c := range key1 {
+		if c == '+' || c == '/' || c == '=' {
+			t.Errorf("Secret key contains non-URL-safe character: %c", c)
+		}
 	}

 	// Check uniqueness
@@ -108,6 +108,8 @@ func (s *WorkerGrpcServer) StartWithTLS(port int) error {
 	go s.cleanupRoutine()
 	go s.activeLogFetchLoop()

+	pb.ServeGrpcOnLocalSocket(grpcServer, port)
+
 	// Start serving in a goroutine
 	go func() {
 		if err := s.grpcServer.Serve(listener); err != nil {
@@ -785,15 +787,6 @@ func (s *WorkerGrpcServer) RequestTaskLogsFromAllWorkers(taskID string, maxEntri
 	return results, nil
 }

-// convertTaskParameters converts task parameters to protobuf format
-func convertTaskParameters(params map[string]interface{}) map[string]string {
-	result := make(map[string]string)
-	for key, value := range params {
-		result[key] = fmt.Sprintf("%v", value)
-	}
-	return result
-}
-
 func findClientAddress(ctx context.Context) string {
 	// fmt.Printf("FromContext %+v\n", ctx)
 	pr, ok := peer.FromContext(ctx)
@@ -312,29 +312,6 @@ func (h *ClusterHandlers) ShowClusterFilers(w http.ResponseWriter, r *http.Reque
 	}
 }

-// ShowClusterBrokers renders the cluster message brokers page
-func (h *ClusterHandlers) ShowClusterBrokers(w http.ResponseWriter, r *http.Request) {
-	// Get cluster brokers data
-	brokersData, err := h.adminServer.GetClusterBrokers()
-	if err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "Failed to get cluster brokers: "+err.Error())
-		return
-	}
-
-	username := usernameOrDefault(r)
-	brokersData.Username = username
-
-	// Render HTML template
-	w.Header().Set("Content-Type", "text/html")
-	brokersComponent := app.ClusterBrokers(*brokersData)
-	viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
-	layoutComponent := layout.Layout(viewCtx, brokersComponent)
-	if err := layoutComponent.Render(r.Context(), w); err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
-		return
-	}
-}
-
 // GetClusterTopology returns the cluster topology as JSON
 func (h *ClusterHandlers) GetClusterTopology(w http.ResponseWriter, r *http.Request) {
 	topology, err := h.adminServer.GetClusterTopology()
@@ -78,34 +78,6 @@ func (h *MessageQueueHandlers) ShowTopics(w http.ResponseWriter, r *http.Request
 	}
 }

-// ShowSubscribers renders the message queue subscribers page
-func (h *MessageQueueHandlers) ShowSubscribers(w http.ResponseWriter, r *http.Request) {
-	// Get subscribers data
-	subscribersData, err := h.adminServer.GetSubscribers()
-	if err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "Failed to get subscribers: "+err.Error())
-		return
-	}
-
-	// Set username
-	username := dash.UsernameFromContext(r.Context())
-	if username == "" {
-		username = "admin"
-	}
-	subscribersData.Username = username
-
-	// Render HTML template
-	w.Header().Set("Content-Type", "text/html")
-	subscribersComponent := app.Subscribers(*subscribersData)
-	viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
-	layoutComponent := layout.Layout(viewCtx, subscribersComponent)
-	err = layoutComponent.Render(r.Context(), w)
-	if err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
-		return
-	}
-}
-
 // ShowTopicDetails renders the topic details page
 func (h *MessageQueueHandlers) ShowTopicDetails(w http.ResponseWriter, r *http.Request) {
 	// Get topic parameters from URL
@@ -93,6 +93,11 @@ func (h *UserHandlers) UpdateUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}

+	if h.adminServer.IsStaticUser(username) {
+		writeJSONError(w, http.StatusForbidden, "Cannot modify static user "+username+" (loaded from config file)")
+		return
+	}
+
 	var req dash.UpdateUserRequest
 	if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
 		writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
@@ -120,6 +125,11 @@ func (h *UserHandlers) DeleteUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}

+	if h.adminServer.IsStaticUser(username) {
+		writeJSONError(w, http.StatusForbidden, "Cannot delete static user "+username+" (loaded from config file)")
+		return
+	}
+
 	err := h.adminServer.DeleteObjectStoreUser(username)
 	if err != nil {
 		glog.Errorf("Failed to delete user %s: %v", username, err)
@@ -1,124 +0,0 @@
-package maintenance
-
-import (
-	"fmt"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
-)
-
-// VerifyProtobufConfig demonstrates that the protobuf configuration system is working
-func VerifyProtobufConfig() error {
-	// Create configuration manager
-	configManager := NewMaintenanceConfigManager()
-	config := configManager.GetConfig()
-
-	// Verify basic configuration
-	if !config.Enabled {
-		return fmt.Errorf("expected config to be enabled by default")
-	}
-
-	if config.ScanIntervalSeconds != 30*60 {
-		return fmt.Errorf("expected scan interval to be 1800 seconds, got %d", config.ScanIntervalSeconds)
-	}
-
-	// Verify policy configuration
-	if config.Policy == nil {
-		return fmt.Errorf("expected policy to be configured")
-	}
-
-	if config.Policy.GlobalMaxConcurrent != 4 {
-		return fmt.Errorf("expected global max concurrent to be 4, got %d", config.Policy.GlobalMaxConcurrent)
-	}
-
-	// Verify task policies
-	vacuumPolicy := config.Policy.TaskPolicies["vacuum"]
-	if vacuumPolicy == nil {
-		return fmt.Errorf("expected vacuum policy to be configured")
-	}
-
-	if !vacuumPolicy.Enabled {
-		return fmt.Errorf("expected vacuum policy to be enabled")
-	}
-
-	// Verify typed configuration access
-	vacuumConfig := vacuumPolicy.GetVacuumConfig()
-	if vacuumConfig == nil {
-		return fmt.Errorf("expected vacuum config to be accessible")
-	}
-
-	if vacuumConfig.GarbageThreshold != 0.3 {
-		return fmt.Errorf("expected garbage threshold to be 0.3, got %f", vacuumConfig.GarbageThreshold)
-	}
-
-	// Verify helper functions work
-	if !IsTaskEnabled(config.Policy, "vacuum") {
-		return fmt.Errorf("expected vacuum task to be enabled via helper function")
-	}
-
-	maxConcurrent := GetMaxConcurrent(config.Policy, "vacuum")
-	if maxConcurrent != 2 {
-		return fmt.Errorf("expected vacuum max concurrent to be 2, got %d", maxConcurrent)
-	}
-
-	// Verify erasure coding configuration
-	ecPolicy := config.Policy.TaskPolicies["erasure_coding"]
-	if ecPolicy == nil {
-		return fmt.Errorf("expected EC policy to be configured")
-	}
-
-	ecConfig := ecPolicy.GetErasureCodingConfig()
-	if ecConfig == nil {
-		return fmt.Errorf("expected EC config to be accessible")
-	}
-
-	// Verify configurable EC fields only
-	if ecConfig.FullnessRatio <= 0 || ecConfig.FullnessRatio > 1 {
-		return fmt.Errorf("expected EC config to have valid fullness ratio (0-1), got %f", ecConfig.FullnessRatio)
-	}
-
-	return nil
-}
-
-// GetProtobufConfigSummary returns a summary of the current protobuf configuration
-func GetProtobufConfigSummary() string {
-	configManager := NewMaintenanceConfigManager()
-	config := configManager.GetConfig()
-
-	summary := fmt.Sprintf("SeaweedFS Protobuf Maintenance Configuration:\n")
-	summary += fmt.Sprintf("  Enabled: %v\n", config.Enabled)
-	summary += fmt.Sprintf("  Scan Interval: %d seconds\n", config.ScanIntervalSeconds)
-	summary += fmt.Sprintf("  Max Retries: %d\n", config.MaxRetries)
-	summary += fmt.Sprintf("  Global Max Concurrent: %d\n", config.Policy.GlobalMaxConcurrent)
-	summary += fmt.Sprintf("  Task Policies: %d configured\n", len(config.Policy.TaskPolicies))
-
-	for taskType, policy := range config.Policy.TaskPolicies {
-		summary += fmt.Sprintf("    %s: enabled=%v, max_concurrent=%d\n",
-			taskType, policy.Enabled, policy.MaxConcurrent)
-	}
-
-	return summary
-}
-
-// CreateCustomConfig demonstrates creating a custom protobuf configuration
-func CreateCustomConfig() *worker_pb.MaintenanceConfig {
-	return &worker_pb.MaintenanceConfig{
-		Enabled:             true,
-		ScanIntervalSeconds: 60 * 60, // 1 hour
-		MaxRetries:          5,
-		Policy: &worker_pb.MaintenancePolicy{
-			GlobalMaxConcurrent: 8,
-			TaskPolicies: map[string]*worker_pb.TaskPolicy{
-				"custom_vacuum": {
-					Enabled:       true,
-					MaxConcurrent: 4,
-					TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
-						VacuumConfig: &worker_pb.VacuumTaskConfig{
-							GarbageThreshold:  0.5,
-							MinVolumeAgeHours: 48,
-						},
-					},
-				},
-			},
-		},
-	}
-}
@@ -1,24 +1,9 @@
 package maintenance

 import (
-	"fmt"
-	"time"
-
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
 )

-// MaintenanceConfigManager handles protobuf-based configuration
-type MaintenanceConfigManager struct {
-	config *worker_pb.MaintenanceConfig
-}
-
-// NewMaintenanceConfigManager creates a new config manager with defaults
-func NewMaintenanceConfigManager() *MaintenanceConfigManager {
-	return &MaintenanceConfigManager{
-		config: DefaultMaintenanceConfigProto(),
-	}
-}
-
 // DefaultMaintenanceConfigProto returns default configuration as protobuf
 func DefaultMaintenanceConfigProto() *worker_pb.MaintenanceConfig {
 	return &worker_pb.MaintenanceConfig{
@@ -34,253 +19,3 @@ func DefaultMaintenanceConfigProto() *worker_pb.MaintenanceConfig {
 		Policy: nil,
 	}
 }
-
-// GetConfig returns the current configuration
-func (mcm *MaintenanceConfigManager) GetConfig() *worker_pb.MaintenanceConfig {
-	return mcm.config
-}
-
-// Type-safe configuration accessors
-
-// GetVacuumConfig returns vacuum-specific configuration for a task type
-func (mcm *MaintenanceConfigManager) GetVacuumConfig(taskType string) *worker_pb.VacuumTaskConfig {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		if vacuumConfig := policy.GetVacuumConfig(); vacuumConfig != nil {
-			return vacuumConfig
-		}
-	}
-	// Return defaults if not configured
-	return &worker_pb.VacuumTaskConfig{
-		GarbageThreshold:  0.3,
-		MinVolumeAgeHours: 24,
-	}
-}
-
-// GetErasureCodingConfig returns EC-specific configuration for a task type
-func (mcm *MaintenanceConfigManager) GetErasureCodingConfig(taskType string) *worker_pb.ErasureCodingTaskConfig {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		if ecConfig := policy.GetErasureCodingConfig(); ecConfig != nil {
-			return ecConfig
-		}
-	}
-	// Return defaults if not configured
-	return &worker_pb.ErasureCodingTaskConfig{
-		FullnessRatio:    0.95,
-		QuietForSeconds:  3600,
-		MinVolumeSizeMb:  100,
-		CollectionFilter: "",
-	}
-}
-
-// GetBalanceConfig returns balance-specific configuration for a task type
-func (mcm *MaintenanceConfigManager) GetBalanceConfig(taskType string) *worker_pb.BalanceTaskConfig {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		if balanceConfig := policy.GetBalanceConfig(); balanceConfig != nil {
-			return balanceConfig
-		}
-	}
-	// Return defaults if not configured
-	return &worker_pb.BalanceTaskConfig{
-		ImbalanceThreshold: 0.2,
-		MinServerCount:     2,
-	}
-}
-
-// GetReplicationConfig returns replication-specific configuration for a task type
-func (mcm *MaintenanceConfigManager) GetReplicationConfig(taskType string) *worker_pb.ReplicationTaskConfig {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		if replicationConfig := policy.GetReplicationConfig(); replicationConfig != nil {
-			return replicationConfig
-		}
-	}
-	// Return defaults if not configured
-	return &worker_pb.ReplicationTaskConfig{
-		TargetReplicaCount: 2,
-	}
-}
-
-// Typed convenience methods for getting task configurations
-
-// GetVacuumTaskConfigForType returns vacuum configuration for a specific task type
-func (mcm *MaintenanceConfigManager) GetVacuumTaskConfigForType(taskType string) *worker_pb.VacuumTaskConfig {
-	return GetVacuumTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
-}
-
-// GetErasureCodingTaskConfigForType returns erasure coding configuration for a specific task type
-func (mcm *MaintenanceConfigManager) GetErasureCodingTaskConfigForType(taskType string) *worker_pb.ErasureCodingTaskConfig {
-	return GetErasureCodingTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
-}
-
-// GetBalanceTaskConfigForType returns balance configuration for a specific task type
-func (mcm *MaintenanceConfigManager) GetBalanceTaskConfigForType(taskType string) *worker_pb.BalanceTaskConfig {
-	return GetBalanceTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
-}
-
-// GetReplicationTaskConfigForType returns replication configuration for a specific task type
-func (mcm *MaintenanceConfigManager) GetReplicationTaskConfigForType(taskType string) *worker_pb.ReplicationTaskConfig {
-	return GetReplicationTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
-}
-
-// Helper methods
-
-func (mcm *MaintenanceConfigManager) getTaskPolicy(taskType string) *worker_pb.TaskPolicy {
-	if mcm.config.Policy != nil && mcm.config.Policy.TaskPolicies != nil {
-		return mcm.config.Policy.TaskPolicies[taskType]
-	}
-	return nil
-}
-
-// IsTaskEnabled returns whether a task type is enabled
-func (mcm *MaintenanceConfigManager) IsTaskEnabled(taskType string) bool {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		return policy.Enabled
-	}
-	return false
-}
-
-// GetMaxConcurrent returns the max concurrent limit for a task type
-func (mcm *MaintenanceConfigManager) GetMaxConcurrent(taskType string) int32 {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		return policy.MaxConcurrent
-	}
-	return 1 // Default
-}
-
-// GetRepeatInterval returns the repeat interval for a task type in seconds
-func (mcm *MaintenanceConfigManager) GetRepeatInterval(taskType string) int32 {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		return policy.RepeatIntervalSeconds
-	}
-	return mcm.config.Policy.DefaultRepeatIntervalSeconds
-}
-
-// GetCheckInterval returns the check interval for a task type in seconds
-func (mcm *MaintenanceConfigManager) GetCheckInterval(taskType string) int32 {
-	if policy := mcm.getTaskPolicy(taskType); policy != nil {
-		return policy.CheckIntervalSeconds
-	}
-	return mcm.config.Policy.DefaultCheckIntervalSeconds
-}
-
-// Duration accessor methods
-
-// GetScanInterval returns the scan interval as a time.Duration
-func (mcm *MaintenanceConfigManager) GetScanInterval() time.Duration {
-	return time.Duration(mcm.config.ScanIntervalSeconds) * time.Second
-}
-
-// GetWorkerTimeout returns the worker timeout as a time.Duration
-func (mcm *MaintenanceConfigManager) GetWorkerTimeout() time.Duration {
-	return time.Duration(mcm.config.WorkerTimeoutSeconds) * time.Second
-}
-
-// GetTaskTimeout returns the task timeout as a time.Duration
-func (mcm *MaintenanceConfigManager) GetTaskTimeout() time.Duration {
-	return time.Duration(mcm.config.TaskTimeoutSeconds) * time.Second
-}
-
-// GetRetryDelay returns the retry delay as a time.Duration
-func (mcm *MaintenanceConfigManager) GetRetryDelay() time.Duration {
-	return time.Duration(mcm.config.RetryDelaySeconds) * time.Second
-}
-
-// GetCleanupInterval returns the cleanup interval as a time.Duration
-func (mcm *MaintenanceConfigManager) GetCleanupInterval() time.Duration {
-	return time.Duration(mcm.config.CleanupIntervalSeconds) * time.Second
-}
-
-// GetTaskRetention returns the task retention period as a time.Duration
-func (mcm *MaintenanceConfigManager) GetTaskRetention() time.Duration {
-	return time.Duration(mcm.config.TaskRetentionSeconds) * time.Second
-}
-
-// ValidateMaintenanceConfigWithSchema validates protobuf maintenance configuration using ConfigField rules
-func ValidateMaintenanceConfigWithSchema(config *worker_pb.MaintenanceConfig) error {
-	if config == nil {
-		return fmt.Errorf("configuration cannot be nil")
-	}
-
-	// Get the schema to access field validation rules
-	schema := GetMaintenanceConfigSchema()
-
-	// Validate each field individually using the ConfigField rules
-	if err := validateFieldWithSchema(schema, "enabled", config.Enabled); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "scan_interval_seconds", int(config.ScanIntervalSeconds)); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "worker_timeout_seconds", int(config.WorkerTimeoutSeconds)); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "task_timeout_seconds", int(config.TaskTimeoutSeconds)); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "retry_delay_seconds", int(config.RetryDelaySeconds)); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "max_retries", int(config.MaxRetries)); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "cleanup_interval_seconds", int(config.CleanupIntervalSeconds)); err != nil {
-		return err
-	}
-
-	if err := validateFieldWithSchema(schema, "task_retention_seconds", int(config.TaskRetentionSeconds)); err != nil {
-		return err
-	}
-
-	// Validate policy fields if present
-	if config.Policy != nil {
-		// Note: These field names might need to be adjusted based on the actual schema
-		if err := validatePolicyField("global_max_concurrent", int(config.Policy.GlobalMaxConcurrent)); err != nil {
-			return err
-		}
-
-		if err := validatePolicyField("default_repeat_interval_seconds", int(config.Policy.DefaultRepeatIntervalSeconds)); err != nil {
-			return err
-		}
-
-		if err := validatePolicyField("default_check_interval_seconds", int(config.Policy.DefaultCheckIntervalSeconds)); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// validateFieldWithSchema validates a single field using its ConfigField definition
-func validateFieldWithSchema(schema *MaintenanceConfigSchema, fieldName string, value interface{}) error {
-	field := schema.GetFieldByName(fieldName)
-	if field == nil {
-		// Field not in schema, skip validation
-		return nil
-	}
-
-	return field.ValidateValue(value)
-}
-
-// validatePolicyField validates policy fields (simplified validation for now)
-func validatePolicyField(fieldName string, value int) error {
-	switch fieldName {
-	case "global_max_concurrent":
-		if value < 1 || value > 20 {
-			return fmt.Errorf("Global Max Concurrent must be between 1 and 20, got %d", value)
-		}
-	case "default_repeat_interval":
-		if value < 1 || value > 168 {
-			return fmt.Errorf("Default Repeat Interval must be between 1 and 168 hours, got %d", value)
-		}
-	case "default_check_interval":
-		if value < 1 || value > 168 {
-			return fmt.Errorf("Default Check Interval must be between 1 and 168 hours, got %d", value)
-		}
-	}
-	return nil
-}
@@ -1055,28 +1055,6 @@ func (mq *MaintenanceQueue) getMaxConcurrentForTaskType(taskType MaintenanceTask
 	return 1
 }

-// getRunningTasks returns all currently running tasks
-func (mq *MaintenanceQueue) getRunningTasks() []*MaintenanceTask {
-	var runningTasks []*MaintenanceTask
-	for _, task := range mq.tasks {
-		if task.Status == TaskStatusAssigned || task.Status == TaskStatusInProgress {
-			runningTasks = append(runningTasks, task)
-		}
-	}
-	return runningTasks
-}
-
-// getAvailableWorkers returns all workers that can take more work
-func (mq *MaintenanceQueue) getAvailableWorkers() []*MaintenanceWorker {
-	var availableWorkers []*MaintenanceWorker
-	for _, worker := range mq.workers {
-		if worker.Status == "active" && worker.CurrentLoad < worker.MaxConcurrent {
-			availableWorkers = append(availableWorkers, worker)
-		}
-	}
-	return availableWorkers
-}
-
 // trackPendingOperation adds a task to the pending operations tracker
 func (mq *MaintenanceQueue) trackPendingOperation(task *MaintenanceTask) {
 	if mq.integration == nil {
@@ -2,15 +2,11 @@ package maintenance

 import (
 	"html/template"
-	"sort"
 	"sync"
 	"time"

-	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
-	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
-	"github.com/seaweedfs/seaweedfs/weed/worker/types"
 )

 // AdminClient interface defines what the maintenance system needs from the admin server
@@ -21,51 +17,6 @@ type AdminClient interface {
 // MaintenanceTaskType represents different types of maintenance operations
 type MaintenanceTaskType string

-// GetRegisteredMaintenanceTaskTypes returns all registered task types as MaintenanceTaskType values
-// sorted alphabetically for consistent menu ordering
-func GetRegisteredMaintenanceTaskTypes() []MaintenanceTaskType {
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-	var taskTypes []MaintenanceTaskType
-
-	for workerTaskType := range typesRegistry.GetAllDetectors() {
-		maintenanceTaskType := MaintenanceTaskType(string(workerTaskType))
-		taskTypes = append(taskTypes, maintenanceTaskType)
-	}
-
-	// Sort task types alphabetically to ensure consistent menu ordering
-	sort.Slice(taskTypes, func(i, j int) bool {
-		return string(taskTypes[i]) < string(taskTypes[j])
-	})
-
-	return taskTypes
-}
-
-// GetMaintenanceTaskType returns a specific task type if it's registered, or empty string if not found
-func GetMaintenanceTaskType(taskTypeName string) MaintenanceTaskType {
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-
-	for workerTaskType := range typesRegistry.GetAllDetectors() {
-		if string(workerTaskType) == taskTypeName {
-			return MaintenanceTaskType(taskTypeName)
-		}
-	}
-
-	return MaintenanceTaskType("")
-}
-
-// IsMaintenanceTaskTypeRegistered checks if a task type is registered
-func IsMaintenanceTaskTypeRegistered(taskType MaintenanceTaskType) bool {
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-
-	for workerTaskType := range typesRegistry.GetAllDetectors() {
-		if string(workerTaskType) == string(taskType) {
-			return true
-		}
-	}
-
-	return false
-}
-
 // MaintenanceTaskPriority represents task execution priority
 type MaintenanceTaskPriority int

@@ -200,14 +151,6 @@ func GetTaskPolicy(mp *MaintenancePolicy, taskType MaintenanceTaskType) *TaskPol
 	return mp.TaskPolicies[string(taskType)]
 }

-// SetTaskPolicy sets the policy for a specific task type
-func SetTaskPolicy(mp *MaintenancePolicy, taskType MaintenanceTaskType, policy *TaskPolicy) {
-	if mp.TaskPolicies == nil {
-		mp.TaskPolicies = make(map[string]*TaskPolicy)
-	}
-	mp.TaskPolicies[string(taskType)] = policy
-}
-
 // IsTaskEnabled returns whether a task type is enabled
 func IsTaskEnabled(mp *MaintenancePolicy, taskType MaintenanceTaskType) bool {
 	policy := GetTaskPolicy(mp, taskType)
@@ -235,84 +178,6 @@ func GetRepeatInterval(mp *MaintenancePolicy, taskType MaintenanceTaskType) int
 	return int(policy.RepeatIntervalSeconds)
 }

-// GetVacuumTaskConfig returns the vacuum task configuration
-func GetVacuumTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.VacuumTaskConfig {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy == nil {
-		return nil
-	}
-	return policy.GetVacuumConfig()
-}
-
-// GetErasureCodingTaskConfig returns the erasure coding task configuration
-func GetErasureCodingTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.ErasureCodingTaskConfig {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy == nil {
-		return nil
-	}
-	return policy.GetErasureCodingConfig()
-}
-
-// GetBalanceTaskConfig returns the balance task configuration
-func GetBalanceTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.BalanceTaskConfig {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy == nil {
-		return nil
-	}
-	return policy.GetBalanceConfig()
-}
-
-// GetReplicationTaskConfig returns the replication task configuration
-func GetReplicationTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.ReplicationTaskConfig {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy == nil {
-		return nil
-	}
-	return policy.GetReplicationConfig()
-}
-
-// Note: GetTaskConfig was removed - use typed getters: GetVacuumTaskConfig, GetErasureCodingTaskConfig, GetBalanceTaskConfig, or GetReplicationTaskConfig
-
-// SetVacuumTaskConfig sets the vacuum task configuration
-func SetVacuumTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.VacuumTaskConfig) {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy != nil {
-		policy.TaskConfig = &worker_pb.TaskPolicy_VacuumConfig{
-			VacuumConfig: config,
-		}
-	}
-}
-
-// SetErasureCodingTaskConfig sets the erasure coding task configuration
-func SetErasureCodingTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.ErasureCodingTaskConfig) {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy != nil {
-		policy.TaskConfig = &worker_pb.TaskPolicy_ErasureCodingConfig{
-			ErasureCodingConfig: config,
-		}
-	}
-}
-
-// SetBalanceTaskConfig sets the balance task configuration
-func SetBalanceTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.BalanceTaskConfig) {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy != nil {
-		policy.TaskConfig = &worker_pb.TaskPolicy_BalanceConfig{
-			BalanceConfig: config,
-		}
-	}
-}
-
-// SetReplicationTaskConfig sets the replication task configuration
-func SetReplicationTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.ReplicationTaskConfig) {
-	policy := GetTaskPolicy(mp, taskType)
-	if policy != nil {
-		policy.TaskConfig = &worker_pb.TaskPolicy_ReplicationConfig{
-			ReplicationConfig: config,
-		}
-	}
-}
-
 // SetTaskConfig sets a configuration value for a task type (legacy method - use typed setters above)
 // Note: SetTaskConfig was removed - use typed setters: SetVacuumTaskConfig, SetErasureCodingTaskConfig, SetBalanceTaskConfig, or SetReplicationTaskConfig

@@ -475,180 +340,6 @@ type ClusterReplicationTask struct {
 	Metadata        map[string]string `json:"metadata,omitempty"`
 }

-// BuildMaintenancePolicyFromTasks creates a maintenance policy with configurations
-// from all registered tasks using their UI providers
-func BuildMaintenancePolicyFromTasks() *MaintenancePolicy {
-	policy := &MaintenancePolicy{
-		TaskPolicies:                 make(map[string]*TaskPolicy),
-		GlobalMaxConcurrent:          4,
-		DefaultRepeatIntervalSeconds: 6 * 3600,  // 6 hours in seconds
-		DefaultCheckIntervalSeconds:  12 * 3600, // 12 hours in seconds
-	}
-
-	// Get all registered task types from the UI registry
-	uiRegistry := tasks.GetGlobalUIRegistry()
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-
-	for taskType, provider := range uiRegistry.GetAllProviders() {
-		// Convert task type to maintenance task type
-		maintenanceTaskType := MaintenanceTaskType(string(taskType))
-
-		// Get the default configuration from the UI provider
-		defaultConfig := provider.GetCurrentConfig()
-
-		// Create task policy from UI configuration
-		taskPolicy := &TaskPolicy{
-			Enabled:               true, // Default enabled
-			MaxConcurrent:         2,    // Default concurrency
-			RepeatIntervalSeconds: policy.DefaultRepeatIntervalSeconds,
-			CheckIntervalSeconds:  policy.DefaultCheckIntervalSeconds,
-		}
-
-		// Extract configuration using TaskConfig interface - no more map conversions!
-		if taskConfig, ok := defaultConfig.(interface{ ToTaskPolicy() *worker_pb.TaskPolicy }); ok {
-			// Use protobuf directly for clean, type-safe config extraction
-			pbTaskPolicy := taskConfig.ToTaskPolicy()
-			taskPolicy.Enabled = pbTaskPolicy.Enabled
-			taskPolicy.MaxConcurrent = pbTaskPolicy.MaxConcurrent
-			if pbTaskPolicy.RepeatIntervalSeconds > 0 {
-				taskPolicy.RepeatIntervalSeconds = pbTaskPolicy.RepeatIntervalSeconds
-			}
-			if pbTaskPolicy.CheckIntervalSeconds > 0 {
-				taskPolicy.CheckIntervalSeconds = pbTaskPolicy.CheckIntervalSeconds
-			}
-		}
-
-		// Also get defaults from scheduler if available (using types.TaskScheduler explicitly)
-		var scheduler types.TaskScheduler = typesRegistry.GetScheduler(taskType)
-		if scheduler != nil {
-			if taskPolicy.MaxConcurrent <= 0 {
-				taskPolicy.MaxConcurrent = int32(scheduler.GetMaxConcurrent())
-			}
-			// Convert default repeat interval to seconds
-			if repeatInterval := scheduler.GetDefaultRepeatInterval(); repeatInterval > 0 {
-				taskPolicy.RepeatIntervalSeconds = int32(repeatInterval.Seconds())
-			}
-		}
-
-		// Also get defaults from detector if available (using types.TaskDetector explicitly)
-		var detector types.TaskDetector = typesRegistry.GetDetector(taskType)
-		if detector != nil {
-			// Convert scan interval to check interval (seconds)
-			if scanInterval := detector.ScanInterval(); scanInterval > 0 {
-				taskPolicy.CheckIntervalSeconds = int32(scanInterval.Seconds())
-			}
-		}
-
-		policy.TaskPolicies[string(maintenanceTaskType)] = taskPolicy
-		glog.V(3).Infof("Built policy for task type %s: enabled=%v, max_concurrent=%d",
-			maintenanceTaskType, taskPolicy.Enabled, taskPolicy.MaxConcurrent)
-	}
-
-	glog.V(2).Infof("Built maintenance policy with %d task configurations", len(policy.TaskPolicies))
-	return policy
-}
-
-// SetPolicyFromTasks sets the maintenance policy from registered tasks
-func SetPolicyFromTasks(policy *MaintenancePolicy) {
-	if policy == nil {
-		return
-	}
-
-	// Build new policy from tasks
-	newPolicy := BuildMaintenancePolicyFromTasks()
-
-	// Copy task policies
-	policy.TaskPolicies = newPolicy.TaskPolicies
-
-	glog.V(1).Infof("Updated maintenance policy with %d task configurations from registered tasks", len(policy.TaskPolicies))
-}
-
-// GetTaskIcon returns the icon CSS class for a task type from its UI provider
-func GetTaskIcon(taskType MaintenanceTaskType) string {
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-	uiRegistry := tasks.GetGlobalUIRegistry()
-
-	// Convert MaintenanceTaskType to TaskType
-	for workerTaskType := range typesRegistry.GetAllDetectors() {
-		if string(workerTaskType) == string(taskType) {
-			// Get the UI provider for this task type
-			provider := uiRegistry.GetProvider(workerTaskType)
-			if provider != nil {
-				return provider.GetIcon()
-			}
-			break
-		}
-	}
-
-	// Default icon if no UI provider found
-	return "fas fa-cog text-muted"
-}
-
-// GetTaskDisplayName returns the display name for a task type from its UI provider
-func GetTaskDisplayName(taskType MaintenanceTaskType) string {
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-	uiRegistry := tasks.GetGlobalUIRegistry()
-
-	// Convert MaintenanceTaskType to TaskType
-	for workerTaskType := range typesRegistry.GetAllDetectors() {
-		if string(workerTaskType) == string(taskType) {
-			// Get the UI provider for this task type
-			provider := uiRegistry.GetProvider(workerTaskType)
-			if provider != nil {
-				return provider.GetDisplayName()
-			}
-			break
-		}
-	}
-
-	// Fallback to the task type string
-	return string(taskType)
-}
-
-// GetTaskDescription returns the description for a task type from its UI provider
-func GetTaskDescription(taskType MaintenanceTaskType) string {
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-	uiRegistry := tasks.GetGlobalUIRegistry()
-
-	// Convert MaintenanceTaskType to TaskType
-	for workerTaskType := range typesRegistry.GetAllDetectors() {
-		if string(workerTaskType) == string(taskType) {
-			// Get the UI provider for this task type
-			provider := uiRegistry.GetProvider(workerTaskType)
-			if provider != nil {
-				return provider.GetDescription()
-			}
-			break
-		}
-	}
-
-	// Fallback to a generic description
-	return "Configure detailed settings for " + string(taskType) + " tasks."
-}
-
-// BuildMaintenanceMenuItems creates menu items for all registered task types
-func BuildMaintenanceMenuItems() []*MaintenanceMenuItem {
-	var menuItems []*MaintenanceMenuItem
-
-	// Get all registered task types
-	registeredTypes := GetRegisteredMaintenanceTaskTypes()
-
-	for _, taskType := range registeredTypes {
-		menuItem := &MaintenanceMenuItem{
-			TaskType:    taskType,
-			DisplayName: GetTaskDisplayName(taskType),
-			Description: GetTaskDescription(taskType),
-			Icon:        GetTaskIcon(taskType),
-			IsEnabled:   IsMaintenanceTaskTypeRegistered(taskType),
-			Path:        "/maintenance/config/" + string(taskType),
-		}
-
-		menuItems = append(menuItems, menuItem)
-	}
-
-	return menuItems
-}
-
 // Helper functions to extract configuration fields

 // Note: Removed getVacuumConfigField, getErasureCodingConfigField, getBalanceConfigField, getReplicationConfigField
@@ -1,421 +0,0 @@
-package maintenance
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"sync"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/worker"
-	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
-	"github.com/seaweedfs/seaweedfs/weed/worker/types"
-
-	// Import task packages to trigger their auto-registration
-	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
-	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
-	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
-)
-
-// MaintenanceWorkerService manages maintenance task execution
-// TaskExecutor defines the function signature for task execution
-type TaskExecutor func(*MaintenanceWorkerService, *MaintenanceTask) error
-
-// TaskExecutorFactory creates a task executor for a given worker service
-type TaskExecutorFactory func() TaskExecutor
-
-// Global registry for task executor factories
-var taskExecutorFactories = make(map[MaintenanceTaskType]TaskExecutorFactory)
-var executorRegistryMutex sync.RWMutex
-var executorRegistryInitOnce sync.Once
-
-// initializeExecutorFactories dynamically registers executor factories for all auto-registered task types
-func initializeExecutorFactories() {
-	executorRegistryInitOnce.Do(func() {
-		// Get all registered task types from the global registry
-		typesRegistry := tasks.GetGlobalTypesRegistry()
-
-		var taskTypes []MaintenanceTaskType
-		for workerTaskType := range typesRegistry.GetAllDetectors() {
-			// Convert types.TaskType to MaintenanceTaskType by string conversion
-			maintenanceTaskType := MaintenanceTaskType(string(workerTaskType))
-			taskTypes = append(taskTypes, maintenanceTaskType)
-		}
-
-		// Register generic executor for all task types
-		for _, taskType := range taskTypes {
-			RegisterTaskExecutorFactory(taskType, createGenericTaskExecutor)
-		}
-
-		glog.V(1).Infof("Dynamically registered generic task executor for %d task types: %v", len(taskTypes), taskTypes)
-	})
-}
-
-// RegisterTaskExecutorFactory registers a factory function for creating task executors
-func RegisterTaskExecutorFactory(taskType MaintenanceTaskType, factory TaskExecutorFactory) {
-	executorRegistryMutex.Lock()
-	defer executorRegistryMutex.Unlock()
-	taskExecutorFactories[taskType] = factory
-	glog.V(2).Infof("Registered executor factory for task type: %s", taskType)
-}
-
-// GetTaskExecutorFactory returns the factory for a task type
-func GetTaskExecutorFactory(taskType MaintenanceTaskType) (TaskExecutorFactory, bool) {
-	// Ensure executor factories are initialized
-	initializeExecutorFactories()
-
-	executorRegistryMutex.RLock()
-	defer executorRegistryMutex.RUnlock()
-	factory, exists := taskExecutorFactories[taskType]
-	return factory, exists
-}
-
-// GetSupportedExecutorTaskTypes returns all task types with registered executor factories
-func GetSupportedExecutorTaskTypes() []MaintenanceTaskType {
-	// Ensure executor factories are initialized
-	initializeExecutorFactories()
-
-	executorRegistryMutex.RLock()
-	defer executorRegistryMutex.RUnlock()
-
-	taskTypes := make([]MaintenanceTaskType, 0, len(taskExecutorFactories))
-	for taskType := range taskExecutorFactories {
-		taskTypes = append(taskTypes, taskType)
-	}
-	return taskTypes
-}
-
-// createGenericTaskExecutor creates a generic task executor that uses the task registry
-func createGenericTaskExecutor() TaskExecutor {
-	return func(mws *MaintenanceWorkerService, task *MaintenanceTask) error {
-		return mws.executeGenericTask(task)
-	}
-}
-
-// init does minimal initialization - actual registration happens lazily
-func init() {
-	// Executor factory registration will happen lazily when first accessed
-	glog.V(1).Infof("Maintenance worker initialized - executor factories will be registered on first access")
-}
-
-type MaintenanceWorkerService struct {
-	workerID      string
-	address       string
-	adminServer   string
-	capabilities  []MaintenanceTaskType
-	maxConcurrent int
-	currentTasks  map[string]*MaintenanceTask
-	queue         *MaintenanceQueue
-	adminClient   AdminClient
-	running       bool
-	stopChan      chan struct{}
-
-	// Task execution registry
-	taskExecutors map[MaintenanceTaskType]TaskExecutor
-
-	// Task registry for creating task instances
-	taskRegistry *tasks.TaskRegistry
-}
-
-// NewMaintenanceWorkerService creates a new maintenance worker service
-func NewMaintenanceWorkerService(workerID, address, adminServer string) *MaintenanceWorkerService {
-	// Get all registered maintenance task types dynamically
-	capabilities := GetRegisteredMaintenanceTaskTypes()
-
-	worker := &MaintenanceWorkerService{
-		workerID:      workerID,
-		address:       address,
-		adminServer:   adminServer,
-		capabilities:  capabilities,
-		maxConcurrent: 2, // Default concurrent task limit
-		currentTasks:  make(map[string]*MaintenanceTask),
-		stopChan:      make(chan struct{}),
-		taskExecutors: make(map[MaintenanceTaskType]TaskExecutor),
-		taskRegistry:  tasks.GetGlobalTaskRegistry(), // Use global registry with auto-registered tasks
-	}
-
-	// Initialize task executor registry
-	worker.initializeTaskExecutors()
-
-	glog.V(1).Infof("Created maintenance worker with %d registered task types", len(worker.taskRegistry.GetAll()))
-
-	return worker
-}
-
-// executeGenericTask executes a task using the task registry instead of hardcoded methods
-func (mws *MaintenanceWorkerService) executeGenericTask(task *MaintenanceTask) error {
-	glog.V(2).Infof("Executing generic task %s: %s for volume %d", task.ID, task.Type, task.VolumeID)
-
-	// Validate that task has proper typed parameters
-	if task.TypedParams == nil {
-		return fmt.Errorf("task %s has no typed parameters - task was not properly planned (insufficient destinations)", task.ID)
-	}
-
-	// Convert MaintenanceTask to types.TaskType
-	taskType := types.TaskType(string(task.Type))
-
-	// Create task instance using the registry
-	taskInstance, err := mws.taskRegistry.Get(taskType).Create(task.TypedParams)
-	if err != nil {
-		return fmt.Errorf("failed to create task instance: %w", err)
-	}
-
-	// Update progress to show task has started
-	mws.updateTaskProgress(task.ID, 5)
-
-	// Execute the task
-	err = taskInstance.Execute(context.Background(), task.TypedParams)
-	if err != nil {
-		return fmt.Errorf("task execution failed: %w", err)
-	}
-
-	// Update progress to show completion
-	mws.updateTaskProgress(task.ID, 100)
-
-	glog.V(2).Infof("Generic task %s completed successfully", task.ID)
-	return nil
-}
-
-// initializeTaskExecutors sets up the task execution registry dynamically
-func (mws *MaintenanceWorkerService) initializeTaskExecutors() {
-	mws.taskExecutors = make(map[MaintenanceTaskType]TaskExecutor)
-
-	// Get all registered executor factories and create executors
-	executorRegistryMutex.RLock()
-	defer executorRegistryMutex.RUnlock()
-
-	for taskType, factory := range taskExecutorFactories {
-		executor := factory()
-		mws.taskExecutors[taskType] = executor
-		glog.V(3).Infof("Initialized executor for task type: %s", taskType)
-	}
-
-	glog.V(2).Infof("Initialized %d task executors", len(mws.taskExecutors))
-}
-
-// RegisterTaskExecutor allows dynamic registration of new task executors
-func (mws *MaintenanceWorkerService) RegisterTaskExecutor(taskType MaintenanceTaskType, executor TaskExecutor) {
-	if mws.taskExecutors == nil {
-		mws.taskExecutors = make(map[MaintenanceTaskType]TaskExecutor)
-	}
-	mws.taskExecutors[taskType] = executor
-	glog.V(1).Infof("Registered executor for task type: %s", taskType)
-}
-
-// GetSupportedTaskTypes returns all task types that this worker can execute
-func (mws *MaintenanceWorkerService) GetSupportedTaskTypes() []MaintenanceTaskType {
-	return GetSupportedExecutorTaskTypes()
-}
-
-// Start begins the worker service
-func (mws *MaintenanceWorkerService) Start() error {
-	mws.running = true
-
-	// Register with admin server
-	worker := &MaintenanceWorker{
-		ID:            mws.workerID,
-		Address:       mws.address,
-		Capabilities:  mws.capabilities,
-		MaxConcurrent: mws.maxConcurrent,
-	}
-
-	if mws.queue != nil {
-		mws.queue.RegisterWorker(worker)
-	}
-
-	// Start worker loop
-	go mws.workerLoop()
-
-	glog.Infof("Maintenance worker %s started at %s", mws.workerID, mws.address)
-	return nil
-}
-
-// Stop terminates the worker service
-func (mws *MaintenanceWorkerService) Stop() {
-	mws.running = false
-	close(mws.stopChan)
-
-	// Wait for current tasks to complete or timeout
-	timeout := time.NewTimer(30 * time.Second)
-	defer timeout.Stop()
-
-	for len(mws.currentTasks) > 0 {
-		select {
-		case <-timeout.C:
-			glog.Warningf("Worker %s stopping with %d tasks still running", mws.workerID, len(mws.currentTasks))
-			return
-		case <-time.After(time.Second):
-			// Check again
-		}
-	}
-
-	glog.Infof("Maintenance worker %s stopped", mws.workerID)
-}
-
-// workerLoop is the main worker event loop
-func (mws *MaintenanceWorkerService) workerLoop() {
-	heartbeatTicker := time.NewTicker(30 * time.Second)
-	defer heartbeatTicker.Stop()
-
-	taskRequestTicker := time.NewTicker(5 * time.Second)
-	defer taskRequestTicker.Stop()
-
-	for mws.running {
-		select {
-		case <-mws.stopChan:
-			return
-		case <-heartbeatTicker.C:
-			mws.sendHeartbeat()
-		case <-taskRequestTicker.C:
-			mws.requestTasks()
-		}
-	}
-}
-
-// sendHeartbeat sends heartbeat to admin server
-func (mws *MaintenanceWorkerService) sendHeartbeat() {
-	if mws.queue != nil {
-		mws.queue.UpdateWorkerHeartbeat(mws.workerID)
-	}
-}
-
-// requestTasks requests new tasks from the admin server
-func (mws *MaintenanceWorkerService) requestTasks() {
-	if len(mws.currentTasks) >= mws.maxConcurrent {
-		return // Already at capacity
-	}
-
-	if mws.queue != nil {
-		task := mws.queue.GetNextTask(mws.workerID, mws.capabilities)
-		if task != nil {
-			mws.executeTask(task)
-		}
-	}
-}
-
-// executeTask executes a maintenance task
-func (mws *MaintenanceWorkerService) executeTask(task *MaintenanceTask) {
-	mws.currentTasks[task.ID] = task
-
-	go func() {
-		defer func() {
-			delete(mws.currentTasks, task.ID)
-		}()
-
-		glog.Infof("Worker %s executing task %s: %s", mws.workerID, task.ID, task.Type)
-
-		// Execute task using dynamic executor registry
-		var err error
-		if executor, exists := mws.taskExecutors[task.Type]; exists {
-			err = executor(mws, task)
-		} else {
-			err = fmt.Errorf("unsupported task type: %s", task.Type)
-			glog.Errorf("No executor registered for task type: %s", task.Type)
-		}
-
-		// Report task completion
-		if mws.queue != nil {
-			errorMsg := ""
-			if err != nil {
-				errorMsg = err.Error()
-			}
-			mws.queue.CompleteTask(task.ID, errorMsg)
-		}
-
-		if err != nil {
-			glog.Errorf("Worker %s failed to execute task %s: %v", mws.workerID, task.ID, err)
-		} else {
-			glog.Infof("Worker %s completed task %s successfully", mws.workerID, task.ID)
-		}
-	}()
-}
-
-// updateTaskProgress updates the progress of a task
-func (mws *MaintenanceWorkerService) updateTaskProgress(taskID string, progress float64) {
-	if mws.queue != nil {
-		mws.queue.UpdateTaskProgress(taskID, progress)
-	}
-}
-
-// GetStatus returns the current status of the worker
-func (mws *MaintenanceWorkerService) GetStatus() map[string]interface{} {
-	return map[string]interface{}{
-		"worker_id":      mws.workerID,
-		"address":        mws.address,
-		"running":        mws.running,
-		"capabilities":   mws.capabilities,
-		"max_concurrent": mws.maxConcurrent,
-		"current_tasks":  len(mws.currentTasks),
-		"task_details":   mws.currentTasks,
-	}
-}
-
-// SetQueue sets the maintenance queue for the worker
-func (mws *MaintenanceWorkerService) SetQueue(queue *MaintenanceQueue) {
-	mws.queue = queue
-}
-
-// SetAdminClient sets the admin client for the worker
-func (mws *MaintenanceWorkerService) SetAdminClient(client AdminClient) {
-	mws.adminClient = client
-}
-
-// SetCapabilities sets the worker capabilities
-func (mws *MaintenanceWorkerService) SetCapabilities(capabilities []MaintenanceTaskType) {
-	mws.capabilities = capabilities
-}
-
-// SetMaxConcurrent sets the maximum concurrent tasks
-func (mws *MaintenanceWorkerService) SetMaxConcurrent(max int) {
-	mws.maxConcurrent = max
-}
-
-// SetHeartbeatInterval sets the heartbeat interval (placeholder for future use)
-func (mws *MaintenanceWorkerService) SetHeartbeatInterval(interval time.Duration) {
-	// Future implementation for configurable heartbeat
-}
-
-// SetTaskRequestInterval sets the task request interval (placeholder for future use)
-func (mws *MaintenanceWorkerService) SetTaskRequestInterval(interval time.Duration) {
-	// Future implementation for configurable task requests
-}
-
-// MaintenanceWorkerCommand represents a standalone maintenance worker command
-type MaintenanceWorkerCommand struct {
-	workerService *MaintenanceWorkerService
-}
-
-// NewMaintenanceWorkerCommand creates a new worker command
-func NewMaintenanceWorkerCommand(workerID, address, adminServer string) *MaintenanceWorkerCommand {
-	return &MaintenanceWorkerCommand{
-		workerService: NewMaintenanceWorkerService(workerID, address, adminServer),
-	}
-}
-
-// Run starts the maintenance worker as a standalone service
-func (mwc *MaintenanceWorkerCommand) Run() error {
-	// Generate or load persistent worker ID if not provided
-	if mwc.workerService.workerID == "" {
-		// Get current working directory for worker ID persistence
-		wd, err := os.Getwd()
-		if err != nil {
-			return fmt.Errorf("failed to get working directory: %w", err)
-		}
-
-		workerID, err := worker.GenerateOrLoadWorkerID(wd)
-		if err != nil {
-			return fmt.Errorf("failed to generate or load worker ID: %w", err)
-		}
-		mwc.workerService.workerID = workerID
-	}
-
-	// Start the worker service
-	err := mwc.workerService.Start()
-	if err != nil {
-		return fmt.Errorf("failed to start maintenance worker: %w", err)
-	}
-
-	// Wait for interrupt signal
-	select {}
-}
@@ -122,6 +122,7 @@ type Plugin struct {
 type streamSession struct {
 	workerID  string
 	outgoing  chan *plugin_pb.AdminToWorkerMessage
+	done      chan struct{}
 	closeOnce sync.Once
 }

@@ -274,6 +275,7 @@ func (r *Plugin) WorkerStream(stream plugin_pb.PluginControlService_WorkerStream
 	session := &streamSession{
 		workerID: workerID,
 		outgoing: make(chan *plugin_pb.AdminToWorkerMessage, r.outgoingBuffer),
+		done:     make(chan struct{}),
 	}
 	r.putSession(session)
 	defer r.cleanupSession(workerID)
@@ -908,8 +910,10 @@ func (r *Plugin) sendLoop(
 			return nil
 		case <-r.shutdownCh:
 			return nil
-		case msg, ok := <-session.outgoing:
-			if !ok {
+		case <-session.done:
+			return nil
+		case msg := <-session.outgoing:
+			if msg == nil {
 				return nil
 			}
 			if err := stream.Send(msg); err != nil {
@@ -930,6 +934,8 @@ func (r *Plugin) sendToWorker(workerID string, message *plugin_pb.AdminToWorkerM
 	select {
 	case <-r.shutdownCh:
 		return fmt.Errorf("plugin is shutting down")
+	case <-session.done:
+		return fmt.Errorf("worker %s session is closed", workerID)
 	case session.outgoing <- message:
 		return nil
 	case <-time.After(r.sendTimeout):
@@ -1425,7 +1431,7 @@ func CloneConfigValueMap(in map[string]*plugin_pb.ConfigValue) map[string]*plugi

 func (s *streamSession) close() {
 	s.closeOnce.Do(func() {
-		close(s.outgoing)
+		close(s.done)
 	})
 }

@@ -26,7 +26,7 @@ func TestRunDetectionSendsCancelOnContextDone(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4)}
+	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4), done: make(chan struct{})}
 	pluginSvc.putSession(session)

 	ctx, cancel := context.WithCancel(context.Background())
@@ -77,7 +77,7 @@ func TestExecuteJobSendsCancelOnContextDone(t *testing.T) {
 			{JobType: jobType, CanExecute: true, MaxExecutionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4)}
+	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4), done: make(chan struct{})}
 	pluginSvc.putSession(session)

 	job := &plugin_pb.JobSpec{JobId: "job-1", JobType: jobType}
@@ -135,8 +135,8 @@ func TestAdminScriptExecutionBlocksOtherDetection(t *testing.T) {
 			{JobType: "vacuum", CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
-	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
+	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
+	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
 	pluginSvc.putSession(adminSession)
 	pluginSvc.putSession(otherSession)

@@ -214,8 +214,8 @@ func TestAdminScriptExecutionBlocksOtherExecution(t *testing.T) {
 			{JobType: "vacuum", CanExecute: true, MaxExecutionConcurrency: 1},
 		},
 	})
-	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
-	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
+	adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
+	otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
 	pluginSvc.putSession(adminSession)
 	pluginSvc.putSession(otherSession)

@@ -22,7 +22,7 @@ func TestRunDetectionIncludesLatestSuccessfulRun(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
+	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
 	pluginSvc.putSession(session)

 	oldSuccess := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
@@ -80,7 +80,7 @@ func TestRunDetectionOmitsLastSuccessfulRunWhenNoSuccessHistory(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
+	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
 	pluginSvc.putSession(session)

 	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
@@ -130,7 +130,7 @@ func TestRunDetectionWithReportCapturesDetectionActivities(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
+	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
 	pluginSvc.putSession(session)

 	reportCh := make(chan *DetectionReport, 1)
@@ -210,7 +210,7 @@ func TestRunDetectionAdminScriptUsesLastCompletedRun(t *testing.T) {
 			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
 		},
 	})
-	session := &streamSession{workerID: "worker-admin-script", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
+	session := &streamSession{workerID: "worker-admin-script", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
 	pluginSvc.putSession(session)

 	successCompleted := time.Date(2026, 2, 1, 10, 0, 0, 0, time.UTC)
@@ -95,16 +95,6 @@ func (r *Plugin) laneSchedulerLoop(ls *schedulerLaneState) {
 	}
 }

-// schedulerLoop is kept for backward compatibility; it delegates to
-// laneSchedulerLoop with the default lane. New code should not call this.
-func (r *Plugin) schedulerLoop() {
-	ls := r.lanes[LaneDefault]
-	if ls == nil {
-		ls = newLaneState(LaneDefault)
-	}
-	r.laneSchedulerLoop(ls)
-}
-
 // runLaneSchedulerIteration runs one scheduling pass for a single lane,
 // processing only the job types assigned to that lane.
 //
@@ -229,82 +219,6 @@ func (r *Plugin) runLaneSchedulerIterationConcurrent(ls *schedulerLaneState, job
 	return hadJobs.Load()
 }

-// runSchedulerIteration is kept for backward compatibility. It runs a
-// single iteration across ALL job types (equivalent to the old single-loop
-// behavior). It is only used by the legacy schedulerLoop() fallback.
-func (r *Plugin) runSchedulerIteration() bool {
-	ls := r.lanes[LaneDefault]
-	if ls == nil {
-		ls = newLaneState(LaneDefault)
-	}
-	// For backward compat, the old function processes all job types.
-	r.expireStaleJobs(time.Now().UTC())
-
-	jobTypes := r.registry.DetectableJobTypes()
-	if len(jobTypes) == 0 {
-		r.setSchedulerLoopState("", "idle")
-		return false
-	}
-
-	r.setSchedulerLoopState("", "waiting_for_lock")
-	releaseLock, err := r.acquireAdminLock("plugin scheduler iteration")
-	if err != nil {
-		glog.Warningf("Plugin scheduler failed to acquire lock: %v", err)
-		r.setSchedulerLoopState("", "idle")
-		return false
-	}
-	if releaseLock != nil {
-		defer releaseLock()
-	}
-
-	active := make(map[string]struct{}, len(jobTypes))
-	hadJobs := false
-
-	for _, jobType := range jobTypes {
-		active[jobType] = struct{}{}
-
-		policy, enabled, err := r.loadSchedulerPolicy(jobType)
-		if err != nil {
-			glog.Warningf("Plugin scheduler failed to load policy for %s: %v", jobType, err)
-			continue
-		}
-		if !enabled {
-			r.clearSchedulerJobType(jobType)
-			continue
-		}
-		initialDelay := time.Duration(0)
-		if runInfo := r.snapshotSchedulerRun(jobType); runInfo.lastRunStartedAt.IsZero() {
-			initialDelay = 5 * time.Second
-		}
-		if !r.markDetectionDue(jobType, policy.DetectionInterval, initialDelay) {
-			continue
-		}
-
-		detected := r.runJobTypeIteration(jobType, policy)
-		if detected {
-			hadJobs = true
-		}
-	}
-
-	r.pruneSchedulerState(active)
-	r.pruneDetectorLeases(active)
-	r.setSchedulerLoopState("", "idle")
-	return hadJobs
-}
-
-// wakeLane wakes the scheduler goroutine for a specific lane.
-func (r *Plugin) wakeLane(lane SchedulerLane) {
-	if r == nil {
-		return
-	}
-	if ls, ok := r.lanes[lane]; ok {
-		select {
-		case ls.wakeCh <- struct{}{}:
-		default:
-		}
-	}
-}
-
 // wakeAllLanes wakes all lane scheduler goroutines.
 func (r *Plugin) wakeAllLanes() {
 	if r == nil {
@@ -210,16 +210,6 @@ func (r *Plugin) setSchedulerLoopStateForJobType(jobType, phase string) {
 	}
 }

-func (r *Plugin) recordSchedulerIterationComplete(hadJobs bool) {
-	if r == nil {
-		return
-	}
-	r.schedulerLoopMu.Lock()
-	r.schedulerLoopState.lastIterationHadJobs = hadJobs
-	r.schedulerLoopState.lastIterationCompleted = time.Now().UTC()
-	r.schedulerLoopMu.Unlock()
-}
-
 func (r *Plugin) snapshotSchedulerLoopState() schedulerLoopState {
 	if r == nil {
 		return schedulerLoopState{}
@@ -125,6 +125,9 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
                                                <div class="d-flex align-items-center">
                                                    <i class="fas fa-user me-2 text-muted"></i>
                                                    <strong>{user.Username}</strong>
+                                                    if user.IsStatic {
+                                                        <span class="badge bg-secondary ms-2" title="Loaded from config file (read-only)">static</span>
+                                                    }
                                                </div>
                                            </td>
                                            <td>{user.Email}</td>
@@ -133,24 +136,28 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
                                            </td>
                                            <td>
                                                <div class="btn-group btn-group-sm" role="group">
-                                                    <button type="button" class="btn btn-outline-info" 
+                                                    <button type="button" class="btn btn-outline-info"
                                                            data-action="show-user-details" data-username={ user.Username }>
                                                        <i class="fas fa-info-circle"></i>
                                                    </button>
-                                                    <button type="button" class="btn btn-outline-primary" 
-                                                            data-action="edit-user" data-username={ user.Username }>
-                                                        <i class="fas fa-edit"></i>
-                                                    </button>
-                                                    if user.Username != "anonymous" {
+                                                    if !user.IsStatic {
+                                                        <button type="button" class="btn btn-outline-primary"
+                                                                data-action="edit-user" data-username={ user.Username }>
+                                                            <i class="fas fa-edit"></i>
+                                                        </button>
+                                                    }
+                                                    if user.Username != "anonymous" && !user.IsStatic {
                                                        <button type="button" class="btn btn-outline-secondary"
                                                                data-action="manage-access-keys" data-username={ user.Username }>
                                                            <i class="fas fa-key"></i>
                                                        </button>
                                                    }
-                                                    <button type="button" class="btn btn-outline-danger" 
-                                                            data-action="delete-user" data-username={ user.Username }>
-                                                        <i class="fas fa-trash"></i>
-                                                    </button>
+                                                    if !user.IsStatic {
+                                                        <button type="button" class="btn btn-outline-danger"
+                                                                data-action="delete-user" data-username={ user.Username }>
+                                                            <i class="fas fa-trash"></i>
+                                                        </button>
+                                                    }
                                                </div>
                                            </td>
                                        </tr>
@@ -163,7 +163,7 @@ templ S3Buckets(data dash.S3BucketsData) {
                                    for _, bucket := range data.Buckets {
                                        <tr>
                                            <td>
-                                                <a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
+                                                <a href={dash.PUrl(ctx, fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
                                                   class="text-decoration-none">
                                                    <i class="fas fa-cube me-2"></i>
                                                    {bucket.Name}
@@ -236,7 +236,7 @@ templ S3Buckets(data dash.S3BucketsData) {
                                            </td>
                                            <td>
                                                <div class="btn-group btn-group-sm" role="group">
-                                                    <a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
+                                                    <a href={dash.PUrl(ctx, fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
                                                       class="btn btn-outline-success btn-sm"
                                                       title="Browse Files">
                                                        <i class="fas fa-folder-open"></i>
@@ -171,9 +171,9 @@ func S3Buckets(data dash.S3BucketsData) templ.Component {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var5 templ.SafeURL
-			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
+			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 166, Col: 123}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 166, Col: 124}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 			if templ_7745c5c3_Err != nil {
@@ -439,9 +439,9 @@ func S3Buckets(data dash.S3BucketsData) templ.Component {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var19 templ.SafeURL
-			templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
+			templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/files?path=/buckets/%s", bucket.Name)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 239, Col: 127}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3_buckets.templ`, Line: 239, Col: 128}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
 			if templ_7745c5c3_Err != nil {
@@ -2,6 +2,7 @@ package app

 import (
 	"fmt"
+	"net/url"

 	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
@@ -152,7 +153,7 @@ templ S3TablesBuckets(data dash.S3TablesBucketsData) {
 												<div class="btn-group btn-group-sm" role="group">
 													{{ bucketName, parseErr := s3tables.ParseBucketNameFromARN(bucket.ARN) }}
 													if parseErr == nil {
-														<a class="btn btn-outline-primary btn-sm" href={ templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)) }>
+														<a class="btn btn-outline-primary btn-sm" href={ dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName))) }>
 															<i class="fas fa-folder-open"></i>
 														</a>
 													} else {
@@ -10,6 +10,7 @@ import templruntime "github.com/a-h/templ/runtime"

 import (
 	"fmt"
+	"net/url"

 	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
@@ -48,7 +49,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var2 templ.SafeURL
 			templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("http://localhost:%d/v1/config", data.IcebergPort)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 23, Col: 124}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 24, Col: 124}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
 			if templ_7745c5c3_Err != nil {
@@ -66,7 +67,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 		var templ_7745c5c3_Var3 string
 		templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 30, Col: 91}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 31, Col: 91}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
 		if templ_7745c5c3_Err != nil {
@@ -84,7 +85,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var4 string
 			templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 37, Col: 109}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 38, Col: 109}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
 			if templ_7745c5c3_Err != nil {
@@ -97,7 +98,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var5 string
 			templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 39, Col: 107}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 40, Col: 107}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 			if templ_7745c5c3_Err != nil {
@@ -120,7 +121,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 		var templ_7745c5c3_Var6 string
 		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalBuckets))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 69, Col: 47}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 70, Col: 47}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 		if templ_7745c5c3_Err != nil {
@@ -133,7 +134,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 		var templ_7745c5c3_Var7 string
 		templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04"))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 88, Col: 54}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 89, Col: 54}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
 		if templ_7745c5c3_Err != nil {
@@ -147,7 +148,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var8 string
 			templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.IcebergPort))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 108, Col: 47}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 109, Col: 47}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
 			if templ_7745c5c3_Err != nil {
@@ -171,7 +172,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var9 string
 			templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 146, Col: 28}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 147, Col: 28}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
 			if templ_7745c5c3_Err != nil {
@@ -184,7 +185,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var10 string
 			templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.OwnerAccountID)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 147, Col: 38}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 148, Col: 38}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
 			if templ_7745c5c3_Err != nil {
@@ -197,7 +198,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var11 string
 			templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 148, Col: 52}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 149, Col: 52}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
 			if templ_7745c5c3_Err != nil {
@@ -210,7 +211,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var12 string
 			templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 149, Col: 52}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 150, Col: 52}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
 			if templ_7745c5c3_Err != nil {
@@ -223,7 +224,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var13 string
 			templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.CreatedAt.Format("2006-01-02 15:04"))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 150, Col: 60}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 151, Col: 60}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
 			if templ_7745c5c3_Err != nil {
@@ -240,9 +241,9 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 					return templ_7745c5c3_Err
 				}
 				var templ_7745c5c3_Var14 templ.SafeURL
-				templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)))
+				templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName))))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 155, Col: 149}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 156, Col: 166}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
 				if templ_7745c5c3_Err != nil {
@@ -265,7 +266,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var15 string
 			templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 163, Col: 122}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 164, Col: 122}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
 			if templ_7745c5c3_Err != nil {
@@ -278,7 +279,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var16 string
 			templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 166, Col: 126}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 167, Col: 126}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
 			if templ_7745c5c3_Err != nil {
@@ -291,7 +292,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var17 string
 			templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.ARN)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 169, Col: 128}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 170, Col: 128}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 			if templ_7745c5c3_Err != nil {
@@ -304,7 +305,7 @@ func S3TablesBuckets(data dash.S3TablesBucketsData) templ.Component {
 			var templ_7745c5c3_Var18 string
 			templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(bucket.Name)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 169, Col: 161}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 170, Col: 161}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
 			if templ_7745c5c3_Err != nil {
@@ -342,7 +343,7 @@ CREATE SECRET (

 SELECT * FROM iceberg_scan('s3://my-table-bucket/my-namespace/my-table');`)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 219, Col: 74}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 220, Col: 74}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
 			if templ_7745c5c3_Err != nil {
@@ -365,7 +366,7 @@ catalog = load_catalog(

 namespaces = catalog.list_namespaces()`)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 235, Col: 39}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 236, Col: 39}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
 			if templ_7745c5c3_Err != nil {
@@ -24,7 +24,7 @@ templ S3TablesTables(data dash.S3TablesTablesData) {
 	<div class="mb-3">
 		{{ bucketName, parseErr := s3tables.ParseBucketNameFromARN(data.BucketARN) }}
 		if parseErr == nil {
-			<a href={ templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)) } class="btn btn-sm btn-outline-secondary">
+			<a href={ dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName))) } class="btn btn-sm btn-outline-secondary">
 				<i class="fas fa-arrow-left me-1"></i>Back to Namespaces
 			</a>
 		} else {
@@ -126,7 +126,7 @@ templ S3TablesTables(data dash.S3TablesTablesData) {
 											<td>
 												<div class="btn-group btn-group-sm" role="group">
 													if parseErr == nil {
-														<a class="btn btn-outline-primary btn-sm" href={ templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables/%s", url.PathEscape(bucketName), url.PathEscape(data.Namespace), url.PathEscape(tableName))) } title="View Iceberg Details">
+														<a class="btn btn-outline-primary btn-sm" href={ dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables/%s", url.PathEscape(bucketName), url.PathEscape(data.Namespace), url.PathEscape(tableName))) } title="View Iceberg Details">
 															<i class="fas fa-eye"></i>
 														</a>
 													} else {
@@ -48,9 +48,9 @@ func S3TablesTables(data dash.S3TablesTablesData) templ.Component {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var2 templ.SafeURL
-			templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)))
+			templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", url.PathEscape(bucketName))))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_tables.templ`, Line: 27, Col: 99}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_tables.templ`, Line: 27, Col: 116}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
 			if templ_7745c5c3_Err != nil {
@@ -308,9 +308,9 @@ func S3TablesTables(data dash.S3TablesTablesData) templ.Component {
 					return templ_7745c5c3_Err
 				}
 				var templ_7745c5c3_Var17 templ.SafeURL
-				templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables/%s", url.PathEscape(bucketName), url.PathEscape(data.Namespace), url.PathEscape(tableName))))
+				templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables/%s", url.PathEscape(bucketName), url.PathEscape(data.Namespace), url.PathEscape(tableName))))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_tables.templ`, Line: 129, Col: 237}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_tables.templ`, Line: 129, Col: 238}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 				if templ_7745c5c3_Err != nil {
@@ -6,20 +6,6 @@ import (
 	"strings"
 )

-// getStatusColor returns Bootstrap color class for status
-func getStatusColor(status string) string {
-	switch status {
-	case "active", "healthy":
-		return "success"
-	case "warning":
-		return "warning"
-	case "critical", "unreachable":
-		return "danger"
-	default:
-		return "secondary"
-	}
-}
-
 // formatBytes converts bytes to human readable format
 func formatBytes(bytes int64) string {
 	if bytes == 0 {
@@ -95,18 +95,6 @@ func NewCluster() *Cluster {
 	}
 }

-func (cluster *Cluster) getGroupMembers(filerGroup FilerGroupName, nodeType string, createIfNotFound bool) *GroupMembers {
-	switch nodeType {
-	case FilerType:
-		return cluster.filerGroups.getGroupMembers(filerGroup, createIfNotFound)
-	case BrokerType:
-		return cluster.brokerGroups.getGroupMembers(filerGroup, createIfNotFound)
-	case S3Type:
-		return cluster.s3Groups.getGroupMembers(filerGroup, createIfNotFound)
-	}
-	return nil
-}
-
 func (cluster *Cluster) AddClusterNode(ns, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
 	filerGroup := FilerGroupName(ns)
 	switch nodeType {
@@ -43,7 +43,8 @@ type LiveLock struct {
 	lc                  *LockClient
 	owner               string
 	lockTTL             time.Duration
-	consecutiveFailures int // Track connection failures to trigger fallback
+	consecutiveFailures int   // Track connection failures to trigger fallback
+	generation          int64 // fencing token from the lock server
 }

 // NewShortLivedLock creates a lock with a 5-second duration
@@ -214,6 +215,9 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
 		glog.V(4).Infof("LOCK: DistributedLock response - key=%s err=%v", lock.key, err)
 		if err == nil && resp != nil {
 			lock.renewToken = resp.RenewToken
+			if resp.Generation > 0 {
+				atomic.StoreInt64(&lock.generation, resp.Generation)
+			}
 			lock.consecutiveFailures = 0 // Reset failure counter on success
 			glog.V(4).Infof("LOCK: Got renewToken for key=%s", lock.key)
 		} else {
@@ -264,6 +268,12 @@ func (lock *LiveLock) LockOwner() string {
 	return lock.owner
 }

+// Generation returns the fencing token for this lock.
+// It increments on each fresh acquisition and stays the same on renewal.
+func (lock *LiveLock) Generation() int64 {
+	return atomic.LoadInt64(&lock.generation)
+}
+
 // IsLocked returns true if this instance currently holds the lock
 func (lock *LiveLock) IsLocked() bool {
 	return atomic.LoadInt32(&lock.isLocked) == 1
@@ -13,10 +13,17 @@ const LiveLockTTL = time.Second * 7

 var NoLockServerError = fmt.Errorf("no lock server found")

+// ReplicateFunc is called to replicate a lock operation to a backup server.
+// The caller (filer server) provides this to avoid a circular dependency.
+// seq is a per-lock monotonic sequence number for causal ordering — the backup
+// rejects mutations with seq <= its current seq for that key.
+type ReplicateFunc func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool)
+
 type DistributedLockManager struct {
-	lockManager *LockManager
-	LockRing    *LockRing
-	Host        pb.ServerAddress
+	lockManager   *LockManager
+	LockRing      *LockRing
+	Host          pb.ServerAddress
+	ReplicateFn   ReplicateFunc // set by filer server after creation
 }

 func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
@@ -27,35 +34,51 @@ func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
 	}
 }

-func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (lockOwner string, renewToken string, movedTo pb.ServerAddress, err error) {
-	movedTo, err = dlm.findLockOwningFiler(key)
-	if err != nil {
-		return
-	}
-	if movedTo != dlm.Host {
-		return
-	}
-	lockOwner, renewToken, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
-	return
-}
-
-func (dlm *DistributedLockManager) findLockOwningFiler(key string) (movedTo pb.ServerAddress, err error) {
-	servers := dlm.LockRing.GetSnapshot()
-	if servers == nil {
+func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (lockOwner string, renewToken string, generation int64, movedTo pb.ServerAddress, err error) {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
 		err = NoLockServerError
 		return
 	}
-
-	movedTo = hashKeyToServer(key, servers)
+	if primary != dlm.Host {
+		// If this is a renewal (non-empty token) and we still hold the lock locally,
+		// serve it here rather than redirecting. This handles the window between
+		// ring update and lock transfer completion — the old primary remains
+		// authoritative for locks it still holds.
+		if token != "" {
+			if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
+				var seq int64
+				lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
+				if err == nil && renewToken != "" {
+					dlm.replicateToBackup(key, expiredAtNs, renewToken, owner, generation, seq, false)
+				}
+				return
+			}
+		}
+		movedTo = primary
+		return
+	}
+	var seq int64
+	lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
+	if err == nil && renewToken != "" {
+		dlm.replicateToBackup(key, expiredAtNs, renewToken, owner, generation, seq, false)
+	}
 	return
 }

 func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, movedTo pb.ServerAddress, err error) {
-	movedTo, err = dlm.findLockOwningFiler(key)
-	if err != nil {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
+		err = NoLockServerError
 		return
 	}
-	if movedTo != dlm.Host {
+	if primary != dlm.Host {
+		// If we still hold this lock locally, serve it here
+		if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup {
+			owner = lock.Owner
+			return
+		}
+		movedTo = primary
 		servers := dlm.LockRing.GetSnapshot()
 		glog.V(0).Infof("lock %s not on current %s but on %s from %v", key, dlm.Host, movedTo, servers)
 		return
@@ -65,26 +88,60 @@ func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, move
 }

 func (dlm *DistributedLockManager) Unlock(key string, token string) (movedTo pb.ServerAddress, err error) {
-	servers := dlm.LockRing.GetSnapshot()
-	if servers == nil {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
 		err = NoLockServerError
 		return
 	}
-
-	server := hashKeyToServer(key, servers)
-	if server != dlm.Host {
-		movedTo = server
+	if primary != dlm.Host {
+		// If we still hold this lock locally, serve the unlock here
+		if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
+			var isUnlocked bool
+			var generation int64
+			var seq int64
+			isUnlocked, generation, seq, err = dlm.lockManager.Unlock(key, token)
+			if isUnlocked {
+				dlm.replicateToBackup(key, 0, "", "", generation, seq, true)
+			}
+			return
+		}
+		movedTo = primary
 		return
 	}
-	_, err = dlm.lockManager.Unlock(key, token)
+	var isUnlocked bool
+	var generation int64
+	var seq int64
+	isUnlocked, generation, seq, err = dlm.lockManager.Unlock(key, token)
+	if isUnlocked {
+		dlm.replicateToBackup(key, 0, "", "", generation, seq, true)
+	}
 	return
 }

-// InsertLock is used to insert a lock to a server unconditionally
-// It is used when a server is down and the lock is moved to another server
-func (dlm *DistributedLockManager) InsertLock(key string, expiredAtNs int64, token string, owner string) {
-	dlm.lockManager.InsertLock(key, expiredAtNs, token, owner)
+// InsertLock is used to insert a lock to a server unconditionally.
+// It is used when a server is down and the lock is moved to another server.
+// After inserting, it replicates to the backup for this key.
+func (dlm *DistributedLockManager) InsertLock(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	if dlm.lockManager.InsertLock(key, expiredAtNs, token, owner, generation, seq) {
+		dlm.replicateToBackup(key, expiredAtNs, token, owner, generation, seq, false)
+	}
 }
+
+// InsertBackupLock inserts a lock as a backup copy, rejecting stale seq
+func (dlm *DistributedLockManager) InsertBackupLock(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	dlm.lockManager.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+}
+
+// RemoveBackupLock removes a backup lock unconditionally
+func (dlm *DistributedLockManager) RemoveBackupLock(key string) {
+	dlm.lockManager.RemoveLock(key)
+}
+
+// RemoveBackupLockIfSeq removes a local copy only if the incoming mutation is not older.
+func (dlm *DistributedLockManager) RemoveBackupLockIfSeq(key string, generation int64, seq int64) {
+	dlm.lockManager.RemoveBackupLockIfSeq(key, generation, seq)
+}
+
 func (dlm *DistributedLockManager) SelectNotOwnedLocks(servers []pb.ServerAddress) (locks []*Lock) {
 	return dlm.lockManager.SelectLocks(func(key string) bool {
 		server := hashKeyToServer(key, servers)
@@ -96,9 +153,40 @@ func (dlm *DistributedLockManager) CalculateTargetServer(key string, servers []p
 }

 func (dlm *DistributedLockManager) IsLocal(key string) bool {
-	servers := dlm.LockRing.GetSnapshot()
-	if len(servers) <= 1 {
+	primary := dlm.LockRing.GetPrimary(key)
+	if primary == "" {
 		return true
 	}
-	return hashKeyToServer(key, servers) == dlm.Host
+	return primary == dlm.Host
+}
+
+// AllLocks returns all non-expired locks on this node
+func (dlm *DistributedLockManager) AllLocks() []*Lock {
+	return dlm.lockManager.AllLocks()
+}
+
+// PromoteLock promotes a backup lock to primary
+func (dlm *DistributedLockManager) PromoteLock(key string) bool {
+	return dlm.lockManager.PromoteLock(key)
+}
+
+// DemoteLock demotes a primary lock to backup
+func (dlm *DistributedLockManager) DemoteLock(key string) bool {
+	return dlm.lockManager.DemoteLock(key)
+}
+
+// GetLock returns a copy of a lock if it exists
+func (dlm *DistributedLockManager) GetLock(key string) (*Lock, bool) {
+	return dlm.lockManager.GetLock(key)
+}
+
+// replicateToBackup asynchronously replicates a lock operation to the backup server
+func (dlm *DistributedLockManager) replicateToBackup(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+	_, backup := dlm.LockRing.GetPrimaryAndBackup(key)
+	if backup == "" {
+		return // single-server deployment, no backup
+	}
+	if dlm.ReplicateFn != nil {
+		go dlm.ReplicateFn(backup, key, expiredAtNs, token, owner, generation, seq, isUnlock)
+	}
 }
@@ -0,0 +1,710 @@
+package lock_manager
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testCluster simulates a cluster of filer nodes with DLMs.
+// It wires up ReplicateFn so that replication calls arrive at the
+// correct peer's DLM, enabling end-to-end backup testing without gRPC.
+type testCluster struct {
+	mu    sync.Mutex
+	nodes map[pb.ServerAddress]*DistributedLockManager
+}
+
+func newTestCluster(hosts ...pb.ServerAddress) *testCluster {
+	c := &testCluster{nodes: make(map[pb.ServerAddress]*DistributedLockManager)}
+	servers := make([]pb.ServerAddress, len(hosts))
+	copy(servers, hosts)
+
+	for _, host := range hosts {
+		dlm := NewDistributedLockManager(host)
+		dlm.LockRing.SetSnapshot(servers, 0)
+		c.nodes[host] = dlm
+	}
+
+	// Wire up replication: each node's ReplicateFn calls the backup's DLM directly
+	for _, dlm := range c.nodes {
+		d := dlm // capture
+		d.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+			c.mu.Lock()
+			target, ok := c.nodes[server]
+			c.mu.Unlock()
+			if !ok {
+				return // server is down
+			}
+			if isUnlock {
+				target.RemoveBackupLockIfSeq(key, generation, seq)
+			} else {
+				target.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+			}
+		}
+	}
+
+	return c
+}
+
+func (c *testCluster) removeNode(host pb.ServerAddress) {
+	c.mu.Lock()
+	delete(c.nodes, host)
+	c.mu.Unlock()
+
+	// Update all remaining nodes' rings
+	remaining := c.getServers()
+	for _, dlm := range c.getNodes() {
+		dlm.LockRing.SetSnapshot(remaining, 0)
+	}
+}
+
+func (c *testCluster) addNode(host pb.ServerAddress) {
+	c.mu.Lock()
+	dlm := NewDistributedLockManager(host)
+	c.nodes[host] = dlm
+	c.mu.Unlock()
+
+	// Wire up replication
+	dlm.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+		c.mu.Lock()
+		target, ok := c.nodes[server]
+		c.mu.Unlock()
+		if !ok {
+			return
+		}
+		if isUnlock {
+			target.RemoveBackupLockIfSeq(key, generation, seq)
+		} else {
+			target.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+		}
+	}
+
+	servers := c.getServers()
+	for _, n := range c.getNodes() {
+		n.LockRing.SetSnapshot(servers, 0)
+	}
+}
+
+func (c *testCluster) getNodes() map[pb.ServerAddress]*DistributedLockManager {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	cp := make(map[pb.ServerAddress]*DistributedLockManager, len(c.nodes))
+	for k, v := range c.nodes {
+		cp[k] = v
+	}
+	return cp
+}
+
+func (c *testCluster) getServers() []pb.ServerAddress {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	var servers []pb.ServerAddress
+	for s := range c.nodes {
+		servers = append(servers, s)
+	}
+	return servers
+}
+
+func (c *testCluster) get(host pb.ServerAddress) *DistributedLockManager {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.nodes[host]
+}
+
+// acquireLock tries to acquire a lock on the correct primary node.
+// It follows redirects (movedTo) like a real client would.
+func (c *testCluster) acquireLock(key, owner string, ttl time.Duration) (renewToken string, generation int64, primaryHost pb.ServerAddress, err error) {
+	// Try any node first (simulates client connecting to seed filer)
+	for _, dlm := range c.getNodes() {
+		expiry := time.Now().Add(ttl).UnixNano()
+		var movedTo pb.ServerAddress
+		var lockErr error
+		_, renewToken, generation, movedTo, lockErr = dlm.LockWithTimeout(key, expiry, "", owner)
+		if movedTo != "" && movedTo != dlm.Host {
+			// Follow redirect
+			target := c.get(movedTo)
+			if target == nil {
+				err = fmt.Errorf("primary %s is down", movedTo)
+				return
+			}
+			_, renewToken, generation, _, lockErr = target.LockWithTimeout(key, expiry, "", owner)
+			if lockErr != nil {
+				err = lockErr
+				return
+			}
+			primaryHost = movedTo
+			// Wait briefly for async replication to complete
+			time.Sleep(10 * time.Millisecond)
+			return
+		}
+		if lockErr != nil {
+			err = lockErr
+			return
+		}
+		primaryHost = dlm.Host
+		time.Sleep(10 * time.Millisecond)
+		return
+	}
+	err = fmt.Errorf("no nodes available")
+	return
+}
+
+// renewLock renews a lock on the primary node
+func (c *testCluster) renewLock(key, owner, token string, ttl time.Duration, primaryHost pb.ServerAddress) (newToken string, generation int64, err error) {
+	target := c.get(primaryHost)
+	if target == nil {
+		err = fmt.Errorf("primary %s is down", primaryHost)
+		return
+	}
+	expiry := time.Now().Add(ttl).UnixNano()
+	var movedTo pb.ServerAddress
+	var lockErr error
+	_, newToken, generation, movedTo, lockErr = target.LockWithTimeout(key, expiry, token, owner)
+	if movedTo != "" && movedTo != primaryHost {
+		target = c.get(movedTo)
+		if target == nil {
+			err = fmt.Errorf("new primary %s is down", movedTo)
+			return
+		}
+		// Pass the existing token so the redirected renewal can match
+		// if the lock was already transferred to the new primary.
+		_, newToken, generation, _, lockErr = target.LockWithTimeout(key, expiry, token, owner)
+	}
+	err = lockErr
+	time.Sleep(10 * time.Millisecond)
+	return
+}
+
+// --- Test Cases ---
+
+func TestDLM_PrimaryCrash_BackupPromotes(t *testing.T) {
+	// Scenario: Lock is acquired, primary crashes, backup should have the lock
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-primary-crash"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+	require.NotEmpty(t, renewToken)
+
+	// Find the backup for this key
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+	require.NotEmpty(t, backup, "should have a backup server")
+
+	// Verify backup has the lock
+	backupDlm := cluster.get(backup)
+	backupLock, found := backupDlm.GetLock(key)
+	require.True(t, found, "backup should have the lock")
+	assert.True(t, backupLock.IsBackup, "lock on backup should be marked as backup")
+	assert.Equal(t, renewToken, backupLock.Token, "backup should have the same token")
+
+	// Crash the primary
+	cluster.removeNode(primaryHost)
+
+	// Simulate topology change: promote backup locks
+	for _, dlm := range cluster.getNodes() {
+		locks := dlm.AllLocks()
+		for _, lock := range locks {
+			newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if newPrimary == dlm.Host && lock.IsBackup {
+				dlm.PromoteLock(lock.Key)
+			}
+		}
+	}
+
+	// The backup should now be the primary
+	newPrimary := backupDlm.LockRing.GetPrimary(key)
+	assert.Equal(t, backup, newPrimary, "backup should be the new primary")
+
+	// The promoted lock should work — verify it's no longer a backup
+	promotedLock, found := backupDlm.GetLock(key)
+	require.True(t, found, "lock should still exist after promotion")
+	assert.False(t, promotedLock.IsBackup, "lock should be promoted to primary")
+
+	// Client should be able to renew with the same token on the new primary
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, backup)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "renewal on new primary should succeed")
+}
+
+func TestDLM_BackupCrash_PrimaryContinues(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-backup-crash"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Crash the backup
+	cluster.removeNode(backup)
+
+	// Primary should still work — renew the lock
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "primary should continue working after backup crash")
+
+	// Verify primary is still the primary for this key
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	assert.Equal(t, primaryHost, newPrimary)
+}
+
+func TestDLM_BothPrimaryAndBackupCrash(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-both-crash"
+	_, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Crash both
+	cluster.removeNode(primaryHost)
+	cluster.removeNode(backup)
+
+	// The lock is lost — the surviving node should be able to acquire it fresh
+	newToken, _, _, err := cluster.acquireLock(key, "owner2", 30*time.Second)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "new owner should acquire lock after both crash")
+}
+
+func TestDLM_RollingRestart(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire multiple locks
+	type lockState struct {
+		key, owner, token string
+		generation        int64
+		primary           pb.ServerAddress
+	}
+	locks := make([]lockState, 5)
+	for i := range locks {
+		key := fmt.Sprintf("rolling-lock-%d", i)
+		token, gen, primary, err := cluster.acquireLock(key, fmt.Sprintf("owner-%d", i), 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockState{key: key, owner: fmt.Sprintf("owner-%d", i), token: token, generation: gen, primary: primary}
+	}
+
+	// Rolling restart: remove and re-add each node one at a time.
+	// After removing a node, promote backups and re-replicate to new backups
+	// to maintain the invariant that each lock has a backup copy.
+	for _, host := range hosts {
+		cluster.removeNode(host)
+
+		// Simulate full OnDlmChangeSnapshot: promote backups and re-replicate
+		for _, dlm := range cluster.getNodes() {
+			for _, lock := range dlm.AllLocks() {
+				newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+				if newPrimary == dlm.Host && lock.IsBackup {
+					dlm.PromoteLock(lock.Key)
+				}
+			}
+			// Re-replicate all primary locks to their new backups
+			for _, lock := range dlm.AllLocks() {
+				newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+				if newPrimary == dlm.Host && !lock.IsBackup {
+					dlm.replicateToBackup(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq, false)
+				}
+			}
+		}
+
+		time.Sleep(10 * time.Millisecond)
+
+		// Re-add the node
+		cluster.addNode(host)
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	// After rolling restart, locks should survive via backup promotion
+	survivedCount := 0
+	for _, ls := range locks {
+		for _, dlm := range cluster.getNodes() {
+			lock, found := dlm.GetLock(ls.key)
+			if found && !lock.IsBackup {
+				survivedCount++
+				break
+			}
+		}
+	}
+	t.Logf("Locks survived rolling restart: %d / %d", survivedCount, len(locks))
+	require.Greater(t, survivedCount, 0, "at least some locks should survive a rolling restart via backup promotion")
+}
+
+func TestDLM_GenerationIncrementsOnNewAcquisition(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "gen-test-lock"
+
+	// Acquire lock — generation should be > 0
+	token1, gen1, primary, err := cluster.acquireLock(key, "owner1", 2*time.Second)
+	require.NoError(t, err)
+	assert.Greater(t, gen1, int64(0))
+
+	// Renew — generation should stay the same
+	token2, gen2, err := cluster.renewLock(key, "owner1", token1, 2*time.Second, primary)
+	require.NoError(t, err)
+	assert.Equal(t, gen1, gen2, "generation should not change on renewal")
+
+	// Let lock expire
+	time.Sleep(3 * time.Second)
+
+	// Re-acquire — generation should increment
+	_, gen3, _, err := cluster.acquireLock(key, "owner2", 30*time.Second)
+	require.NoError(t, err)
+	assert.Greater(t, gen3, gen1, "generation should increment on new acquisition")
+	_ = token2
+}
+
+func TestDLM_ReplicationFailure_PrimaryStillWorks(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Break replication by setting a no-op ReplicateFn on all nodes
+	for _, dlm := range cluster.getNodes() {
+		dlm.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+			// Simulate replication failure: do nothing
+		}
+	}
+
+	key := "repl-fail-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	// Primary should have the lock
+	primaryDlm := cluster.get(primaryHost)
+	lock, found := primaryDlm.GetLock(key)
+	require.True(t, found, "primary should have the lock")
+	assert.False(t, lock.IsBackup)
+
+	// Backup should NOT have it (replication failed)
+	_, backup := primaryDlm.LockRing.GetPrimaryAndBackup(key)
+	backupDlm := cluster.get(backup)
+	_, found = backupDlm.GetLock(key)
+	assert.False(t, found, "backup should not have the lock when replication fails")
+
+	// Primary should still be able to renew
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken)
+}
+
+func TestDLM_UnlockReplicatesToBackup(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "unlock-repl-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Verify backup has the lock
+	_, found := cluster.get(backup).GetLock(key)
+	require.True(t, found, "backup should have the lock")
+
+	// Unlock on primary
+	primaryDlm := cluster.get(primaryHost)
+	movedTo, err := primaryDlm.Unlock(key, renewToken)
+	require.NoError(t, err)
+	assert.Empty(t, movedTo)
+
+	// Wait for async replication
+	time.Sleep(20 * time.Millisecond)
+
+	// Backup should also have removed the lock
+	_, found = cluster.get(backup).GetLock(key)
+	assert.False(t, found, "backup should remove lock after unlock replication")
+}
+
+func TestDLM_TopologyChange_LockSurvivesServerAddition(t *testing.T) {
+	// Start with 2 servers, acquire lock, add a 3rd server
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "topo-add-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	// Add a new server
+	cluster.addNode("filer3:8888")
+	time.Sleep(20 * time.Millisecond)
+
+	// The lock should still be accessible — either the same primary or on a new one
+	// Try to renew on the original primary first
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	if newPrimary == primaryHost {
+		// Still on same primary
+		newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+		require.NoError(t, err)
+		assert.NotEmpty(t, newToken)
+	}
+	// If primary changed, the lock may need transfer — that's handled by OnDlmChangeSnapshot
+	// which is tested at the server level
+}
+
+func TestDLM_ConsistentHashing_MinimalDisruption(t *testing.T) {
+	// Verify that removing a server only affects locks on that server
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire 50 locks
+	type lockInfo struct {
+		key, token string
+		primary    pb.ServerAddress
+	}
+	locks := make([]lockInfo, 50)
+	for i := range locks {
+		key := fmt.Sprintf("min-disrupt-%d", i)
+		token, _, primary, err := cluster.acquireLock(key, "owner", 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockInfo{key: key, token: token, primary: primary}
+	}
+
+	// Count locks per server before removal
+	countBefore := make(map[pb.ServerAddress]int)
+	for _, l := range locks {
+		countBefore[l.primary]++
+	}
+	t.Logf("Lock distribution before: %v", countBefore)
+
+	// Remove filer2
+	cluster.removeNode("filer2:8888")
+
+	// Count how many locks changed primary
+	changed := 0
+	for _, l := range locks {
+		// Check where the lock should be now
+		for _, dlm := range cluster.getNodes() {
+			newPrimary := dlm.LockRing.GetPrimary(l.key)
+			if newPrimary != l.primary {
+				changed++
+			}
+			break
+		}
+	}
+
+	// Only locks from filer2 should have changed
+	assert.Equal(t, countBefore["filer2:8888"], changed,
+		"only locks from removed server should change primary")
+}
+
+func TestDLM_NodeDropAndJoin_OwnershipDisruption(t *testing.T) {
+	// Scenario: 3 nodes, acquire locks, one drops and a NEW node joins quickly.
+	// The new node steals hash ranges from surviving nodes, not just from the
+	// departed node. This test measures the disruption.
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire many locks
+	numLocks := 100
+	type lockInfo struct {
+		key, token string
+		primary    pb.ServerAddress
+	}
+	locks := make([]lockInfo, numLocks)
+	for i := range locks {
+		key := fmt.Sprintf("churn-lock-%d", i)
+		token, _, primary, err := cluster.acquireLock(key, "owner", 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockInfo{key: key, token: token, primary: primary}
+	}
+
+	// Record primary for each lock before the change
+	beforePrimary := make(map[string]pb.ServerAddress)
+	for _, l := range locks {
+		beforePrimary[l.key] = l.primary
+	}
+
+	// Drop filer3 and immediately add filer4
+	cluster.removeNode("filer3:8888")
+
+	// Promote backups on remaining nodes (simulates OnDlmChangeSnapshot)
+	for _, dlm := range cluster.getNodes() {
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p == dlm.Host && lock.IsBackup {
+				dlm.PromoteLock(lock.Key)
+			}
+		}
+		// Re-replicate primary locks to new backups
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p == dlm.Host && !lock.IsBackup {
+				dlm.replicateToBackup(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq, false)
+			}
+		}
+	}
+	time.Sleep(10 * time.Millisecond)
+
+	// Now add filer4 (new node, empty)
+	cluster.addNode("filer4:8888")
+	time.Sleep(10 * time.Millisecond)
+
+	// Simulate OnDlmChangeSnapshot on all nodes after filer4 joins:
+	// transfer locks that now belong to filer4
+	for host, dlm := range cluster.getNodes() {
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p != host && !lock.IsBackup {
+				// This lock should move to the new primary
+				target := cluster.get(p)
+				if target != nil {
+					target.InsertLock(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq)
+					dlm.DemoteLock(lock.Key)
+				}
+			}
+		}
+	}
+	time.Sleep(10 * time.Millisecond)
+
+	// Count disruptions: locks whose primary changed to a node other than filer3's successor
+	disruptedFromSurvivors := 0
+	disruptedFromDeparted := 0
+	movedToFiler4 := 0
+	for _, l := range locks {
+		// What's the new primary?
+		var newPrimary pb.ServerAddress
+		for _, dlm := range cluster.getNodes() {
+			newPrimary = dlm.LockRing.GetPrimary(l.key)
+			break
+		}
+		oldPrimary := beforePrimary[l.key]
+		if newPrimary != oldPrimary {
+			if oldPrimary == "filer3:8888" {
+				disruptedFromDeparted++
+			} else {
+				disruptedFromSurvivors++
+			}
+		}
+		if newPrimary == "filer4:8888" {
+			movedToFiler4++
+		}
+	}
+
+	t.Logf("Locks disrupted from departed filer3: %d / %d", disruptedFromDeparted, numLocks)
+	t.Logf("Locks disrupted from surviving filer1/filer2: %d / %d", disruptedFromSurvivors, numLocks)
+	t.Logf("Locks now on new filer4: %d / %d", movedToFiler4, numLocks)
+
+	// The key concern: filer4 joining disrupts locks on surviving nodes
+	// With consistent hashing, new node steals ~1/N of each surviving node's keys
+	// Verify that the transfer logic above moved those locks to filer4
+	for _, l := range locks {
+		var newPrimary pb.ServerAddress
+		for _, dlm := range cluster.getNodes() {
+			newPrimary = dlm.LockRing.GetPrimary(l.key)
+			break
+		}
+		target := cluster.get(newPrimary)
+		require.NotNil(t, target, "primary %s should exist", newPrimary)
+
+		lock, found := target.GetLock(l.key)
+		if !found {
+			// Lock may have only a backup copy if transfer happened but
+			// the lock was on the departed node and wasn't re-replicated.
+			// Check all nodes for any copy.
+			anyFound := false
+			for _, dlm := range cluster.getNodes() {
+				if _, f := dlm.GetLock(l.key); f {
+					anyFound = true
+					break
+				}
+			}
+			if !anyFound {
+				t.Errorf("lock %s completely lost (primary should be %s)", l.key, newPrimary)
+			}
+			continue
+		}
+		assert.False(t, lock.IsBackup, "lock %s on primary %s should not be a backup", l.key, newPrimary)
+	}
+}
+
+func TestDLM_RenewalDuringTransferWindow(t *testing.T) {
+	// When a new node joins and steals a key range from a surviving node,
+	// there's a window between ring update and lock transfer. During this
+	// window, a client renewal should still succeed on the old primary
+	// (because it still holds the lock locally).
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Find a key that will move primary when filer4 is added.
+	// Try candidate keys until we find one whose primary changes.
+	var key, renewToken string
+	var primaryHost pb.ServerAddress
+	for i := 0; i < 1000; i++ {
+		candidate := fmt.Sprintf("transfer-window-lock-%d", i)
+		token, _, primary, err := cluster.acquireLock(candidate, "owner1", 30*time.Second)
+		require.NoError(t, err)
+
+		// Check if adding filer4 would move this key's primary
+		tmpRing := NewHashRing(DefaultVnodeCount)
+		tmpRing.SetServers([]pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888", "filer4:8888"})
+		newPrimary := tmpRing.GetPrimary(candidate)
+		if newPrimary != primary {
+			key = candidate
+			renewToken = token
+			primaryHost = primary
+			break
+		}
+	}
+	require.NotEmpty(t, key, "should find a key that moves primary when filer4 joins")
+
+	// Add filer4 — this changes the primary for our key per the ring
+	cluster.addNode("filer4:8888")
+	time.Sleep(10 * time.Millisecond)
+
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	require.NotEqual(t, primaryHost, newPrimary, "key should have moved to a different primary")
+
+	// Renewal on the OLD primary should still succeed because it holds the lock locally
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err, "renewal on old primary should succeed during transfer window")
+	assert.NotEmpty(t, newToken, "should get a new token from old primary")
+	t.Logf("Key %s: primary changed from %s to %s, but renewal on old primary succeeded", key, primaryHost, newPrimary)
+}
+
+func TestDLM_StaleReplicationRejected(t *testing.T) {
+	// Verify that a stale replication (lower seq) does not overwrite a newer one
+	lm := NewLockManager()
+
+	// Insert backup with seq=3
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 1, 3)
+	lock, found := lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(3), lock.Seq)
+
+	// Try to overwrite with stale seq=2 — should be rejected
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 1, 2)
+	lock, found = lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-new", lock.Token, "stale replication should be rejected")
+	assert.Equal(t, int64(3), lock.Seq)
+
+	// Update with higher seq=4 — should succeed
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-newer", "owner1", 1, 4)
+	lock, found = lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-newer", lock.Token, "newer replication should be accepted")
+	assert.Equal(t, int64(4), lock.Seq)
+
+	// Stale unlock (seq=2) should not delete the lock
+	removed := lm.RemoveBackupLockIfSeq("key1", 1, 2)
+	assert.False(t, removed, "stale unlock should be rejected")
+	_, found = lm.GetLock("key1")
+	assert.True(t, found, "lock should still exist after stale unlock")
+
+	// Valid unlock (seq=5) should delete
+	removed = lm.RemoveBackupLockIfSeq("key1", 1, 5)
+	assert.True(t, removed, "valid unlock should be accepted")
+	_, found = lm.GetLock("key1")
+	assert.False(t, found, "lock should be removed after valid unlock")
+}
@@ -0,0 +1,198 @@
+package lock_manager
+
+import (
+	"hash/crc32"
+	"sort"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+const DefaultVnodeCount = 50
+
+// HashRing implements consistent hashing with virtual nodes.
+// When a server is removed, only the keys that hashed to that server
+// are remapped (to the next server on the ring), leaving all other
+// key-to-server mappings stable.
+//
+// UPGRADE NOTE: This replaces the previous modulo-based hashing
+// (hash % len(servers)). The two schemes compute different primaries
+// for the same key, so all filer nodes in the cluster must be upgraded
+// together (or via a rolling restart that completes within the lock TTL
+// window of ~7 seconds) to avoid routing disagreements.
+type HashRing struct {
+	mu            sync.RWMutex
+	vnodeCount    int
+	sortedHashes  []uint32                     // sorted ring positions
+	vnodeToServer map[uint32]pb.ServerAddress   // ring position → server
+	servers       map[pb.ServerAddress]struct{} // set of all servers
+}
+
+func NewHashRing(vnodeCount int) *HashRing {
+	if vnodeCount <= 0 {
+		vnodeCount = DefaultVnodeCount
+	}
+	return &HashRing{
+		vnodeCount:    vnodeCount,
+		vnodeToServer: make(map[uint32]pb.ServerAddress),
+		servers:       make(map[pb.ServerAddress]struct{}),
+	}
+}
+
+// AddServer adds a server with virtual nodes to the ring.
+func (hr *HashRing) AddServer(server pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	if _, exists := hr.servers[server]; exists {
+		return
+	}
+	hr.servers[server] = struct{}{}
+	hr.rebuildRing()
+}
+
+// RemoveServer removes a server and its virtual nodes from the ring.
+func (hr *HashRing) RemoveServer(server pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	if _, exists := hr.servers[server]; !exists {
+		return
+	}
+	delete(hr.servers, server)
+	hr.rebuildRing()
+}
+
+// SetServers replaces the entire server set.
+func (hr *HashRing) SetServers(servers []pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	hr.servers = make(map[pb.ServerAddress]struct{}, len(servers))
+	for _, s := range servers {
+		hr.servers[s] = struct{}{}
+	}
+	hr.rebuildRing()
+}
+
+// GetPrimaryAndBackup returns the primary server for a key and its backup
+// (the next distinct server clockwise on the ring).
+// If there is only one server, backup is empty.
+func (hr *HashRing) GetPrimaryAndBackup(key string) (primary, backup pb.ServerAddress) {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	if len(hr.sortedHashes) == 0 {
+		return "", ""
+	}
+
+	hash := hashKey(key)
+	idx := hr.search(hash)
+	primary = hr.vnodeToServer[hr.sortedHashes[idx]]
+
+	// Walk clockwise to find a different server for backup
+	ringLen := len(hr.sortedHashes)
+	for i := 1; i < ringLen; i++ {
+		candidate := hr.vnodeToServer[hr.sortedHashes[(idx+i)%ringLen]]
+		if candidate != primary {
+			backup = candidate
+			return
+		}
+	}
+	// Only one server — no backup
+	return primary, ""
+}
+
+// GetPrimary returns just the primary server for a key.
+func (hr *HashRing) GetPrimary(key string) pb.ServerAddress {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	if len(hr.sortedHashes) == 0 {
+		return ""
+	}
+
+	hash := hashKey(key)
+	idx := hr.search(hash)
+	return hr.vnodeToServer[hr.sortedHashes[idx]]
+}
+
+// GetServers returns a sorted copy of all servers in the ring.
+func (hr *HashRing) GetServers() []pb.ServerAddress {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	servers := make([]pb.ServerAddress, 0, len(hr.servers))
+	for s := range hr.servers {
+		servers = append(servers, s)
+	}
+	sort.Slice(servers, func(i, j int) bool {
+		return servers[i] < servers[j]
+	})
+	return servers
+}
+
+// ServerCount returns the number of servers in the ring.
+func (hr *HashRing) ServerCount() int {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+	return len(hr.servers)
+}
+
+// rebuildRing rebuilds the sorted hash ring from the current server set.
+// Caller must hold hr.mu write lock.
+func (hr *HashRing) rebuildRing() {
+	hr.vnodeToServer = make(map[uint32]pb.ServerAddress, len(hr.servers)*hr.vnodeCount)
+	hr.sortedHashes = make([]uint32, 0, len(hr.servers)*hr.vnodeCount)
+
+	for server := range hr.servers {
+		for i := 0; i < hr.vnodeCount; i++ {
+			vnodeKey := vnodeKeyFor(server, i)
+			hash := hashKey(vnodeKey)
+			hr.vnodeToServer[hash] = server
+			hr.sortedHashes = append(hr.sortedHashes, hash)
+		}
+	}
+	sort.Slice(hr.sortedHashes, func(i, j int) bool {
+		return hr.sortedHashes[i] < hr.sortedHashes[j]
+	})
+}
+
+// search finds the first ring position >= hash.
+func (hr *HashRing) search(hash uint32) int {
+	idx := sort.Search(len(hr.sortedHashes), func(i int) bool {
+		return hr.sortedHashes[i] >= hash
+	})
+	if idx >= len(hr.sortedHashes) {
+		idx = 0 // wrap around
+	}
+	return idx
+}
+
+func hashKey(key string) uint32 {
+	return crc32.ChecksumIEEE([]byte(key))
+}
+
+func vnodeKeyFor(server pb.ServerAddress, index int) string {
+	// Use a format that distributes well across the ring
+	buf := make([]byte, 0, len(server)+10)
+	buf = append(buf, []byte(server)...)
+	buf = append(buf, '#')
+	buf = appendInt(buf, index)
+	return string(buf)
+}
+
+func appendInt(buf []byte, n int) []byte {
+	if n == 0 {
+		return append(buf, '0')
+	}
+	// Simple int-to-string without importing strconv
+	digits := [20]byte{}
+	pos := len(digits)
+	for n > 0 {
+		pos--
+		digits[pos] = byte('0' + n%10)
+		n /= 10
+	}
+	return append(buf, digits[pos:]...)
+}
@@ -0,0 +1,173 @@
+package lock_manager
+
+import (
+	"fmt"
+	"math"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHashRing_BasicOperations(t *testing.T) {
+	hr := NewHashRing(50)
+
+	// Empty ring
+	p, b := hr.GetPrimaryAndBackup("key1")
+	assert.Equal(t, pb.ServerAddress(""), p)
+	assert.Equal(t, pb.ServerAddress(""), b)
+
+	// Single server — no backup
+	hr.AddServer("filer1:8888")
+	p, b = hr.GetPrimaryAndBackup("key1")
+	assert.Equal(t, pb.ServerAddress("filer1:8888"), p)
+	assert.Equal(t, pb.ServerAddress(""), b)
+
+	// Two servers — backup is the other server
+	hr.AddServer("filer2:8888")
+	p, b = hr.GetPrimaryAndBackup("key1")
+	assert.NotEqual(t, p, b)
+	assert.NotEmpty(t, b)
+
+	// Three servers
+	hr.AddServer("filer3:8888")
+	p, b = hr.GetPrimaryAndBackup("key1")
+	assert.NotEqual(t, p, b)
+	assert.NotEmpty(t, b)
+
+	// Remove server
+	hr.RemoveServer("filer2:8888")
+	assert.Equal(t, 2, hr.ServerCount())
+}
+
+func TestHashRing_DuplicateAddRemove(t *testing.T) {
+	hr := NewHashRing(50)
+
+	hr.AddServer("filer1:8888")
+	hr.AddServer("filer1:8888") // duplicate
+	assert.Equal(t, 1, hr.ServerCount())
+
+	hr.RemoveServer("filer1:8888")
+	assert.Equal(t, 0, hr.ServerCount())
+
+	hr.RemoveServer("filer1:8888") // remove non-existent
+	assert.Equal(t, 0, hr.ServerCount())
+}
+
+func TestHashRing_SetServers(t *testing.T) {
+	hr := NewHashRing(50)
+
+	hr.SetServers([]pb.ServerAddress{"a:1", "b:2", "c:3"})
+	assert.Equal(t, 3, hr.ServerCount())
+
+	servers := hr.GetServers()
+	assert.Equal(t, 3, len(servers))
+
+	// SetServers replaces
+	hr.SetServers([]pb.ServerAddress{"x:1", "y:2"})
+	assert.Equal(t, 2, hr.ServerCount())
+}
+
+func TestHashRing_ConsistencyOnRemoval(t *testing.T) {
+	// The key property of consistent hashing: when a server is removed,
+	// only keys that mapped to the removed server change.
+	hr := NewHashRing(50)
+	servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	hr.SetServers(servers)
+
+	numKeys := 1000
+	// Record where each key maps before removal
+	before := make(map[string]pb.ServerAddress, numKeys)
+	for i := 0; i < numKeys; i++ {
+		key := fmt.Sprintf("lock-key-%d", i)
+		before[key] = hr.GetPrimary(key)
+	}
+
+	// Remove filer2
+	hr.RemoveServer("filer2:8888")
+
+	moved := 0
+	for i := 0; i < numKeys; i++ {
+		key := fmt.Sprintf("lock-key-%d", i)
+		after := hr.GetPrimary(key)
+		if before[key] != after {
+			// Only keys from filer2 should move
+			assert.Equal(t, pb.ServerAddress("filer2:8888"), before[key],
+				"key %s moved from %s to %s, but it wasn't on the removed server", key, before[key], after)
+			moved++
+		}
+	}
+	// Roughly 1/3 of keys should move (those that were on filer2)
+	t.Logf("Keys that moved: %d / %d", moved, numKeys)
+	assert.Greater(t, moved, 0, "some keys should have moved")
+	assert.Less(t, moved, numKeys, "not all keys should move")
+}
+
+func TestHashRing_BackupIsSuccessor(t *testing.T) {
+	// After removing primary, the backup should become the new primary
+	hr := NewHashRing(50)
+	servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	hr.SetServers(servers)
+
+	// For each key, verify that removing the primary makes the backup the new primary
+	promoted := 0
+	total := 500
+	for i := 0; i < total; i++ {
+		key := fmt.Sprintf("test-lock-%d", i)
+		primary, backup := hr.GetPrimaryAndBackup(key)
+		assert.NotEqual(t, primary, backup)
+
+		// Temporarily remove primary
+		hr.RemoveServer(primary)
+		newPrimary := hr.GetPrimary(key)
+		if newPrimary == backup {
+			promoted++
+		}
+		// Restore
+		hr.AddServer(primary)
+	}
+	// The backup should become new primary for all keys
+	assert.Equal(t, total, promoted,
+		"backup should become new primary for all keys when primary is removed")
+}
+
+func TestHashRing_Distribution(t *testing.T) {
+	hr := NewHashRing(50)
+	servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	hr.SetServers(servers)
+
+	counts := make(map[pb.ServerAddress]int)
+	numKeys := 3000
+	for i := 0; i < numKeys; i++ {
+		key := fmt.Sprintf("dist-key-%d", i)
+		p := hr.GetPrimary(key)
+		counts[p]++
+	}
+
+	expected := float64(numKeys) / float64(len(servers))
+	for server, count := range counts {
+		deviation := math.Abs(float64(count)-expected) / expected
+		t.Logf("Server %s: %d keys (%.1f%% deviation)", server, count, deviation*100)
+		// Allow up to 40% deviation with 50 vnodes and 3 servers
+		assert.Less(t, deviation, 0.40,
+			"server %s has too many or too few keys: %d (expected ~%d)", server, count, int(expected))
+	}
+}
+
+func TestHashRing_GetPrimary(t *testing.T) {
+	hr := NewHashRing(50)
+
+	// Empty ring
+	assert.Equal(t, pb.ServerAddress(""), hr.GetPrimary("key"))
+
+	hr.SetServers([]pb.ServerAddress{"a:1", "b:2"})
+
+	// Deterministic: same key always maps to same server
+	p1 := hr.GetPrimary("mykey")
+	p2 := hr.GetPrimary("mykey")
+	assert.Equal(t, p1, p2)
+
+	// GetPrimary matches the primary from GetPrimaryAndBackup
+	primary, _ := hr.GetPrimaryAndBackup("mykey")
+	assert.Equal(t, primary, hr.GetPrimary("mykey"))
+}
@@ -3,6 +3,7 @@ package lock_manager
 import (
 	"fmt"
 	"sync"
+	"sync/atomic"
 	"time"

 	"github.com/google/uuid"
@@ -17,14 +18,18 @@ var LockNotFound = fmt.Errorf("lock not found")

 // LockManager local lock manager, used by distributed lock manager
 type LockManager struct {
-	locks      map[string]*Lock
-	accessLock sync.RWMutex
+	locks            map[string]*Lock
+	accessLock       sync.RWMutex
+	nextGeneration   atomic.Int64
 }
 type Lock struct {
 	Token       string
 	ExpiredAtNs int64
 	Key         string // only used for moving locks
 	Owner       string
+	IsBackup    bool   // true if this node holds the lock as a backup
+	Generation  int64  // monotonic fencing token, increments on fresh acquisition
+	Seq         int64  // per-lock sequence number, increments on every mutation (acquire/renew/unlock)
 }

 func NewLockManager() *LockManager {
@@ -35,7 +40,38 @@ func NewLockManager() *LockManager {
 	return t
 }

-func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner string) (lockOwner, renewToken string, err error) {
+func (lm *LockManager) NextGeneration() int64 {
+	return lm.nextGeneration.Add(1)
+}
+
+func compareMutationVersion(generation, seq int64, existingGeneration, existingSeq int64) int {
+	switch {
+	case generation < existingGeneration:
+		return -1
+	case generation > existingGeneration:
+		return 1
+	case seq < existingSeq:
+		return -1
+	case seq > existingSeq:
+		return 1
+	default:
+		return 0
+	}
+}
+
+func (lm *LockManager) advanceGenerationFloor(generation int64) {
+	for {
+		current := lm.nextGeneration.Load()
+		if generation < current {
+			return
+		}
+		if lm.nextGeneration.CompareAndSwap(current, generation) {
+			return
+		}
+	}
+}
+
+func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner string) (lockOwner, renewToken string, generation int64, seq int64, err error) {
 	lm.accessLock.Lock()
 	defer lm.accessLock.Unlock()

@@ -51,8 +87,10 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
 			} else {
 				// new lock
 				renewToken = uuid.New().String()
-				glog.V(4).Infof("key %s new token %v owner %v", path, renewToken, owner)
-				lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
+				generation = lm.NextGeneration()
+				seq = 1
+				glog.V(4).Infof("key %s new token %v owner %v generation %d", path, renewToken, owner, generation)
+				lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
 				return
 			}
 		}
@@ -61,8 +99,10 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
 		if oldValue.Token == token {
 			// token matches, renew the lock
 			renewToken = uuid.New().String()
+			generation = oldValue.Generation // keep same generation on renewal
+			seq = oldValue.Seq + 1
 			glog.V(4).Infof("key %s old token %v owner %v => %v owner %v", path, oldValue.Token, oldValue.Owner, renewToken, owner)
-			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
+			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
 			return
 		} else {
 			if token == "" {
@@ -79,9 +119,11 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
 		glog.V(4).Infof("key %s no lock owner %v", path, owner)
 		if token == "" {
 			// new lock
-			glog.V(4).Infof("key %s new token %v owner %v", path, token, owner)
 			renewToken = uuid.New().String()
-			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
+			generation = lm.NextGeneration()
+			seq = 1
+			glog.V(4).Infof("key %s new token %v owner %v generation %d", path, renewToken, owner, generation)
+			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
 			return
 		} else {
 			glog.V(4).Infof("key %s non-empty token %v owner %v", path, token, owner)
@@ -91,7 +133,7 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
 	}
 }

-func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, err error) {
+func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, generation int64, seq int64, err error) {
 	lm.accessLock.Lock()
 	defer lm.accessLock.Unlock()

@@ -100,12 +142,16 @@ func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, err e
 		if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < now.UnixNano() {
 			// lock is expired, delete it
 			isUnlocked = true
+			generation = oldValue.Generation
+			seq = oldValue.Seq + 1
 			glog.V(4).Infof("key %s expired at %v", path, time.Unix(0, oldValue.ExpiredAtNs))
 			delete(lm.locks, path)
 			return
 		}
 		if oldValue.Token == token {
 			isUnlocked = true
+			generation = oldValue.Generation
+			seq = oldValue.Seq + 1
 			glog.V(4).Infof("key %s unlocked with %v", path, token)
 			delete(lm.locks, path)
 			return
@@ -140,10 +186,10 @@ func (lm *LockManager) CleanUp() {
 }

 // SelectLocks takes out locks by key
-// if keyFn return true, the lock will be taken out
+// if selectFn returns true, the lock will be removed and returned
 func (lm *LockManager) SelectLocks(selectFn func(key string) bool) (locks []*Lock) {
-	lm.accessLock.RLock()
-	defer lm.accessLock.RUnlock()
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()

 	now := time.Now().UnixNano()

@@ -163,12 +209,90 @@ func (lm *LockManager) SelectLocks(selectFn func(key string) bool) (locks []*Loc
 	return
 }

-// InsertLock inserts a lock unconditionally
-func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string, owner string) {
+// InsertLock inserts a lock from a transfer if it is newer than the current state.
+func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string, owner string, generation int64, seq int64) bool {
 	lm.accessLock.Lock()
 	defer lm.accessLock.Unlock()

-	lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner}
+	if existing, found := lm.locks[path]; found {
+		if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) <= 0 {
+			glog.V(4).Infof("lock %s: rejecting stale transfer gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
+			return false
+		}
+	}
+	lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
+	lm.advanceGenerationFloor(generation)
+	return true
+}
+
+// InsertBackupLock inserts or updates a lock as a backup copy.
+// It rejects stale mutations by comparing (generation, seq). If a current primary
+// already exists on this node, newer replicated state refreshes that primary copy
+// without demoting it back to a backup.
+func (lm *LockManager) InsertBackupLock(path string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if existing, found := lm.locks[path]; found {
+		if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) <= 0 {
+			glog.V(4).Infof("backup lock %s: rejecting stale gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
+			return
+		}
+		lm.locks[path] = &Lock{
+			Token:       token,
+			ExpiredAtNs: expiredAtNs,
+			Owner:       owner,
+			IsBackup:    existing.IsBackup,
+			Generation:  generation,
+			Seq:         seq,
+		}
+		lm.advanceGenerationFloor(generation)
+		return
+	}
+	lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner, IsBackup: true, Generation: generation, Seq: seq}
+	lm.advanceGenerationFloor(generation)
+}
+
+// RemoveLock removes a lock by key
+func (lm *LockManager) RemoveLock(path string) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+	delete(lm.locks, path)
+}
+
+// RemoveBackupLockIfSeq removes the local copy only if the incoming mutation is
+// not older than the current (generation, seq). This prevents a late unlock from
+// deleting a newer reacquired lock whose seq has reset.
+func (lm *LockManager) RemoveBackupLockIfSeq(path string, generation int64, seq int64) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if existing, found := lm.locks[path]; found {
+		if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) >= 0 {
+			delete(lm.locks, path)
+			return true
+		}
+		glog.V(4).Infof("backup lock %s: rejecting stale unlock gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
+		return false
+	}
+	return true
+}
+
+// GetLock returns a copy of the lock for a key, if it exists and is not expired
+func (lm *LockManager) GetLock(key string) (*Lock, bool) {
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
+
+	lock, found := lm.locks[key]
+	if !found {
+		return nil, false
+	}
+	if time.Now().UnixNano() > lock.ExpiredAtNs {
+		return nil, false
+	}
+	// Return a copy
+	cp := *lock
+	return &cp, true
 }

 func (lm *LockManager) GetLockOwner(key string) (owner string, err error) {
@@ -176,8 +300,54 @@ func (lm *LockManager) GetLockOwner(key string) (owner string, err error) {
 	defer lm.accessLock.RUnlock()

 	if lock, found := lm.locks[key]; found {
+		if time.Now().UnixNano() > lock.ExpiredAtNs {
+			err = LockNotFound
+			return
+		}
 		return lock.Owner, nil
 	}
 	err = LockNotFound
 	return
 }
+
+// AllLocks returns a copy of all non-expired locks
+func (lm *LockManager) AllLocks() []*Lock {
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
+
+	now := time.Now().UnixNano()
+	var result []*Lock
+	for key, lock := range lm.locks {
+		if now > lock.ExpiredAtNs {
+			continue
+		}
+		cp := *lock
+		cp.Key = key
+		result = append(result, &cp)
+	}
+	return result
+}
+
+// PromoteLock changes a backup lock to a primary lock
+func (lm *LockManager) PromoteLock(key string) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if lock, found := lm.locks[key]; found && lock.IsBackup {
+		lock.IsBackup = false
+		return true
+	}
+	return false
+}
+
+// DemoteLock changes a primary lock to a backup lock
+func (lm *LockManager) DemoteLock(key string) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if lock, found := lm.locks[key]; found && !lock.IsBackup {
+		lock.IsBackup = true
+		return true
+	}
+	return false
+}
@@ -0,0 +1,86 @@
+package lock_manager
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLockManager_GetLockOwnerIgnoresExpiredLock(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertLock("key1", time.Now().Add(-time.Second).UnixNano(), "token1", "owner1", 7, 3)
+
+	owner, err := lm.GetLockOwner("key1")
+	assert.Empty(t, owner)
+	assert.ErrorIs(t, err, LockNotFound)
+}
+
+func TestLockManager_InsertLockRejectsStaleTransfer(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 4)
+	lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 7, 3)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(4), lock.Seq)
+}
+
+func TestLockManager_InsertLockAdvancesGenerationCounter(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token1", "owner1", 12, 1)
+
+	_, renewToken, generation, _, err := lm.Lock("key2", time.Now().Add(30*time.Second).UnixNano(), "", "owner2")
+	assert.NoError(t, err)
+	assert.NotEmpty(t, renewToken)
+	assert.Greater(t, generation, int64(12))
+}
+
+func TestLockManager_InsertBackupLockRejectsOlderGeneration(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 1)
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 7, 9)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(1), lock.Seq)
+}
+
+func TestLockManager_InsertBackupLockKeepsPrimaryRole(t *testing.T) {
+	lm := NewLockManager()
+
+	ok := lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 8, 1)
+	assert.True(t, ok)
+
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 2)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.False(t, lock.IsBackup)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(2), lock.Seq)
+}
+
+func TestLockManager_RemoveBackupLockRejectsOlderGeneration(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 1)
+
+	removed := lm.RemoveBackupLockIfSeq("key1", 7, 9)
+	assert.False(t, removed)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(1), lock.Seq)
+}
@@ -7,7 +7,6 @@ import (

 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 )

 type LockRingSnapshot struct {
@@ -18,18 +17,18 @@ type LockRingSnapshot struct {
 type LockRing struct {
 	sync.RWMutex
 	snapshots        []*LockRingSnapshot
-	candidateServers map[pb.ServerAddress]struct{}
-	lastUpdateTime   time.Time
 	lastCompactTime  time.Time
 	snapshotInterval time.Duration
 	onTakeSnapshot   func(snapshot []pb.ServerAddress)
 	cleanupWg        sync.WaitGroup
+	Ring             *HashRing // consistent hash ring
+	version          int64    // monotonic version from master, rejects stale updates
 }

 func NewLockRing(snapshotInterval time.Duration) *LockRing {
 	return &LockRing{
 		snapshotInterval: snapshotInterval,
-		candidateServers: make(map[pb.ServerAddress]struct{}),
+		Ring:             NewHashRing(DefaultVnodeCount),
 	}
 }

@@ -39,52 +38,26 @@ func (r *LockRing) SetTakeSnapshotCallback(onTakeSnapshot func(snapshot []pb.Ser
 	r.onTakeSnapshot = onTakeSnapshot
 }

-// AddServer adds a server to the ring
-// if the previous snapshot passed the snapshot interval, create a new snapshot
-func (r *LockRing) AddServer(server pb.ServerAddress) {
-	glog.V(0).Infof("add server %v", server)
-	r.Lock()
-
-	if _, found := r.candidateServers[server]; found {
-		glog.V(0).Infof("add server: already exists %v", server)
-		r.Unlock()
-		return
-	}
-	r.lastUpdateTime = time.Now()
-	r.candidateServers[server] = struct{}{}
-	r.Unlock()
-
-	r.takeSnapshotWithDelayedCompaction()
-}
-
-func (r *LockRing) RemoveServer(server pb.ServerAddress) {
-	glog.V(0).Infof("remove server %v", server)
-
-	r.Lock()
-
-	if _, found := r.candidateServers[server]; !found {
-		r.Unlock()
-		return
-	}
-	r.lastUpdateTime = time.Now()
-	delete(r.candidateServers, server)
-	r.Unlock()
-
-	r.takeSnapshotWithDelayedCompaction()
-}
-
-func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
+// SetSnapshot replaces the ring with a new server list from the master.
+// The version must be >= the current version, otherwise the update is rejected
+// (protects against reordered messages). Version 0 is always accepted (bootstrap).
+func (r *LockRing) SetSnapshot(servers []pb.ServerAddress, version int64) bool {

 	sort.Slice(servers, func(i, j int) bool {
 		return servers[i] < servers[j]
 	})

 	r.Lock()
-	r.lastUpdateTime = time.Now()
-	// init candidateServers
-	for _, server := range servers {
-		r.candidateServers[server] = struct{}{}
+	if version > 0 && version < r.version {
+		glog.V(0).Infof("LockRing: rejecting stale update v%d (current v%d)", version, r.version)
+		r.Unlock()
+		return false
 	}
+	r.version = version
+	// Update the ring while holding the lock so version and ring state
+	// are always consistent — prevents a concurrent SetSnapshot from
+	// seeing the new version but applying its servers to the old ring.
+	r.Ring.SetServers(servers)
 	r.Unlock()

 	r.addOneSnapshot(servers)
@@ -95,23 +68,14 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
 		<-time.After(r.snapshotInterval)
 		r.compactSnapshots()
 	}()
+	return true
 }

-func (r *LockRing) takeSnapshotWithDelayedCompaction() {
-	r.doTakeSnapshot()
-
-	r.cleanupWg.Add(1)
-	go func() {
-		defer r.cleanupWg.Done()
-		<-time.After(r.snapshotInterval)
-		r.compactSnapshots()
-	}()
-}
-
-func (r *LockRing) doTakeSnapshot() {
-	servers := r.getSortedServers()
-
-	r.addOneSnapshot(servers)
+// Version returns the current ring version.
+func (r *LockRing) Version() int64 {
+	r.RLock()
+	defer r.RUnlock()
+	return r.version
 }

 func (r *LockRing) addOneSnapshot(servers []pb.ServerAddress) {
@@ -138,35 +102,19 @@ func (r *LockRing) compactSnapshots() {
 	r.Lock()
 	defer r.Unlock()

-	// Always attempt compaction when called, regardless of lastCompactTime
-	// This ensures proper cleanup even with multiple concurrent compaction requests
-
 	ts := time.Now()
-	// remove old snapshots
 	recentSnapshotIndex := 1
 	for ; recentSnapshotIndex < len(r.snapshots); recentSnapshotIndex++ {
 		if ts.Sub(r.snapshots[recentSnapshotIndex].ts) > r.snapshotInterval {
 			break
 		}
 	}
-	// keep the one that has been running for a while
 	if recentSnapshotIndex+1 <= len(r.snapshots) {
 		r.snapshots = r.snapshots[:recentSnapshotIndex+1]
 	}
 	r.lastCompactTime = ts
 }

-func (r *LockRing) getSortedServers() []pb.ServerAddress {
-	sortedServers := make([]pb.ServerAddress, 0, len(r.candidateServers))
-	for server := range r.candidateServers {
-		sortedServers = append(sortedServers, server)
-	}
-	sort.Slice(sortedServers, func(i, j int) bool {
-		return sortedServers[i] < sortedServers[j]
-	})
-	return sortedServers
-}
-
 func (r *LockRing) GetSnapshot() (servers []pb.ServerAddress) {
 	r.RLock()
 	defer r.RUnlock()
@@ -178,7 +126,6 @@ func (r *LockRing) GetSnapshot() (servers []pb.ServerAddress) {
 }

 // WaitForCleanup waits for all pending cleanup operations to complete
-// This is useful for testing to ensure deterministic behavior
 func (r *LockRing) WaitForCleanup() {
 	r.cleanupWg.Wait()
 }
@@ -190,14 +137,23 @@ func (r *LockRing) GetSnapshotCount() int {
 	return len(r.snapshots)
 }

+// GetPrimaryAndBackup returns the primary and backup servers for a key
+// using the consistent hash ring.
+func (r *LockRing) GetPrimaryAndBackup(key string) (primary, backup pb.ServerAddress) {
+	return r.Ring.GetPrimaryAndBackup(key)
+}
+
+// GetPrimary returns the primary server for a key using the consistent hash ring.
+func (r *LockRing) GetPrimary(key string) pb.ServerAddress {
+	return r.Ring.GetPrimary(key)
+}
+
+// hashKeyToServer uses a temporary consistent hash ring for the given server list.
 func hashKeyToServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
 	if len(servers) == 0 {
 		return ""
 	}
-	x := util.HashStringToLong(key)
-	if x < 0 {
-		x = -x
-	}
-	x = x % int64(len(servers))
-	return servers[x]
+	ring := NewHashRing(DefaultVnodeCount)
+	ring.SetServers(servers)
+	return ring.GetPrimary(key)
 }
@@ -8,84 +8,91 @@ import (
 	"github.com/stretchr/testify/assert"
 )

-func TestAddServer(t *testing.T) {
+func TestLockRing_SetSnapshot(t *testing.T) {
 	r := NewLockRing(100 * time.Millisecond)

-	// Add servers
-	r.AddServer("localhost:8080")
-	r.AddServer("localhost:8081")
-	r.AddServer("localhost:8082")
-	r.AddServer("localhost:8083")
-	r.AddServer("localhost:8084")
+	// Set 5 servers
+	r.SetSnapshot([]pb.ServerAddress{
+		"localhost:8080", "localhost:8081", "localhost:8082",
+		"localhost:8083", "localhost:8084",
+	}, 1)

-	// Verify all servers are present
 	servers := r.GetSnapshot()
 	assert.Equal(t, 5, len(servers))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8080"))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8081"))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8082"))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8083"))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8084"))

-	// Remove servers
-	r.RemoveServer("localhost:8084")
-	r.RemoveServer("localhost:8082")
-	r.RemoveServer("localhost:8080")
+	// Replace with 2 servers
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8081", "localhost:8083"}, 2)

-	// Wait for all cleanup operations to complete
 	r.WaitForCleanup()
-
-	// Verify only 2 servers remain (localhost:8081 and localhost:8083)
 	servers = r.GetSnapshot()
 	assert.Equal(t, 2, len(servers))
 	assert.Contains(t, servers, pb.ServerAddress("localhost:8081"))
 	assert.Contains(t, servers, pb.ServerAddress("localhost:8083"))

-	// Verify cleanup has happened - wait for snapshot interval and check snapshots are compacted
+	// Verify compaction
 	time.Sleep(110 * time.Millisecond)
 	r.WaitForCleanup()
-
-	// Verify snapshot history is cleaned up properly (should have at most 2 snapshots after compaction)
-	snapshotCount := r.GetSnapshotCount()
-	assert.LessOrEqual(t, snapshotCount, 2, "Snapshot history should be compacted")
+	assert.LessOrEqual(t, r.GetSnapshotCount(), 2)
 }

-func TestLockRing(t *testing.T) {
+func TestLockRing_SnapshotCompaction(t *testing.T) {
 	r := NewLockRing(100 * time.Millisecond)

-	// Test initial snapshot
-	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081"})
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081"}, 1)
 	assert.Equal(t, 1, r.GetSnapshotCount())
-	servers := r.GetSnapshot()
-	assert.Equal(t, 2, len(servers))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8080"))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8081"))

-	// Add another server
-	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082"})
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082"}, 2)
 	assert.Equal(t, 2, r.GetSnapshotCount())
-	servers = r.GetSnapshot()
-	assert.Equal(t, 3, len(servers))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8082"))

-	// Wait for cleanup interval and add another server
+	// Wait for compaction
 	time.Sleep(110 * time.Millisecond)
 	r.WaitForCleanup()
-	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082", "localhost:8083"})
-	assert.LessOrEqual(t, r.GetSnapshotCount(), 3)
-	servers = r.GetSnapshot()
-	assert.Equal(t, 4, len(servers))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8083"))

-	// Wait for cleanup and verify compaction
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082", "localhost:8083"}, 3)
+	assert.LessOrEqual(t, r.GetSnapshotCount(), 3)
+	servers := r.GetSnapshot()
+	assert.Equal(t, 4, len(servers))
+
 	time.Sleep(110 * time.Millisecond)
 	r.WaitForCleanup()
 	assert.LessOrEqual(t, r.GetSnapshotCount(), 2, "Snapshots should be compacted")

-	// Add final server
-	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082", "localhost:8083", "localhost:8084"})
+	r.SetSnapshot([]pb.ServerAddress{
+		"localhost:8080", "localhost:8081", "localhost:8082",
+		"localhost:8083", "localhost:8084",
+	}, 4)
 	servers = r.GetSnapshot()
 	assert.Equal(t, 5, len(servers))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8084"))
-	assert.LessOrEqual(t, r.GetSnapshotCount(), 3)
+}
+
+func TestLockRing_VersionRejectsStale(t *testing.T) {
+	r := NewLockRing(100 * time.Millisecond)
+
+	// Apply version 3
+	ok := r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2", "c:3"}, 3)
+	assert.True(t, ok)
+	assert.Equal(t, int64(3), r.Version())
+	assert.Equal(t, 3, len(r.GetSnapshot()))
+
+	// Stale version 2 — should be rejected
+	ok = r.SetSnapshot([]pb.ServerAddress{"x:1"}, 2)
+	assert.False(t, ok)
+	assert.Equal(t, int64(3), r.Version())
+	assert.Equal(t, 3, len(r.GetSnapshot()), "stale update should not change the ring")
+
+	// Same version 3 — accepted (SetSnapshot accepts version >= current, state-changing)
+	ok = r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2"}, 3)
+	assert.True(t, ok)
+	assert.Equal(t, 2, len(r.GetSnapshot()))
+
+	// Newer version 5 — should be accepted
+	ok = r.SetSnapshot([]pb.ServerAddress{"d:1", "e:2", "f:3", "g:4"}, 5)
+	assert.True(t, ok)
+	assert.Equal(t, int64(5), r.Version())
+	assert.Equal(t, 4, len(r.GetSnapshot()))
+
+	// Version 0 always accepted (bootstrap)
+	ok = r.SetSnapshot([]pb.ServerAddress{"z:1"}, 0)
+	assert.True(t, ok)
+	assert.Equal(t, 1, len(r.GetSnapshot()))
 }
@@ -0,0 +1,164 @@
+package cluster
+
+import (
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+const LockRingStabilizationInterval = 1 * time.Second
+
+// LockRingManager tracks filer membership for the distributed lock ring.
+// It batches rapid topology changes (e.g., node drop + join) with a
+// stabilization timer, then broadcasts the complete member list atomically
+// so filers receive a single consistent ring update instead of multiple
+// intermediate states.
+type LockRingManager struct {
+	mu              sync.Mutex
+	members         map[FilerGroupName]map[pb.ServerAddress]struct{}
+	version         map[FilerGroupName]int64
+	lastBroadcast   map[FilerGroupName]*master_pb.LockRingUpdate
+	pendingTimer    map[FilerGroupName]*time.Timer
+	broadcastFn     func(resp *master_pb.KeepConnectedResponse)
+	stabilizeDelay  time.Duration
+}
+
+func NewLockRingManager(broadcastFn func(resp *master_pb.KeepConnectedResponse)) *LockRingManager {
+	return &LockRingManager{
+		members:        make(map[FilerGroupName]map[pb.ServerAddress]struct{}),
+		version:        make(map[FilerGroupName]int64),
+		lastBroadcast:  make(map[FilerGroupName]*master_pb.LockRingUpdate),
+		pendingTimer:   make(map[FilerGroupName]*time.Timer),
+		broadcastFn:    broadcastFn,
+		stabilizeDelay: LockRingStabilizationInterval,
+	}
+}
+
+// AddServer records a filer joining and schedules a batched broadcast.
+func (lrm *LockRingManager) AddServer(filerGroup FilerGroupName, address pb.ServerAddress) {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+
+	if _, ok := lrm.members[filerGroup]; !ok {
+		lrm.members[filerGroup] = make(map[pb.ServerAddress]struct{})
+	}
+	lrm.members[filerGroup][address] = struct{}{}
+	lrm.scheduleBroadcast(filerGroup)
+}
+
+// RemoveServer records a filer leaving and schedules a batched broadcast.
+func (lrm *LockRingManager) RemoveServer(filerGroup FilerGroupName, address pb.ServerAddress) {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+
+	if members, ok := lrm.members[filerGroup]; ok {
+		delete(members, address)
+	}
+	lrm.scheduleBroadcast(filerGroup)
+}
+
+// GetServers returns the current member list for a filer group.
+func (lrm *LockRingManager) GetServers(filerGroup FilerGroupName) []string {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+
+	members, ok := lrm.members[filerGroup]
+	if !ok {
+		return nil
+	}
+	servers := make([]string, 0, len(members))
+	for addr := range members {
+		servers = append(servers, string(addr))
+	}
+	return servers
+}
+
+// GetVersion returns the current version for a filer group.
+func (lrm *LockRingManager) GetVersion(filerGroup FilerGroupName) int64 {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+	return lrm.version[filerGroup]
+}
+
+// GetLastUpdate returns a copy of the most recently broadcast lock-ring snapshot
+// for the filer group. It intentionally does not expose pending, unstabilized changes.
+func (lrm *LockRingManager) GetLastUpdate(filerGroup FilerGroupName) *master_pb.LockRingUpdate {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+
+	update, ok := lrm.lastBroadcast[filerGroup]
+	if !ok || update == nil {
+		return nil
+	}
+	cp := *update
+	cp.Servers = append([]string(nil), update.Servers...)
+	return &cp
+}
+
+// scheduleBroadcast resets the stabilization timer. If another change arrives
+// before the timer fires, the timer resets, batching the changes.
+// Caller must hold lrm.mu.
+func (lrm *LockRingManager) scheduleBroadcast(filerGroup FilerGroupName) {
+	if timer, ok := lrm.pendingTimer[filerGroup]; ok {
+		if !timer.Stop() {
+			// Timer already fired, callback is running or queued.
+			// It will pick up the latest state from lrm.members, so
+			// just schedule a new one for any further changes.
+		}
+	}
+	lrm.pendingTimer[filerGroup] = time.AfterFunc(lrm.stabilizeDelay, func() {
+		lrm.doBroadcast(filerGroup)
+	})
+}
+
+func (lrm *LockRingManager) doBroadcast(filerGroup FilerGroupName) {
+	lrm.mu.Lock()
+	// Use wall-clock nanoseconds so the version survives master restarts
+	// without persistence — a restarted master produces a version greater
+	// than any pre-restart value (assuming clocks don't jump backward).
+	version := time.Now().UnixNano()
+	lrm.version[filerGroup] = version
+	servers := make([]string, 0)
+	if members, ok := lrm.members[filerGroup]; ok {
+		for addr := range members {
+			servers = append(servers, string(addr))
+		}
+	}
+	update := &master_pb.LockRingUpdate{
+		FilerGroup: string(filerGroup),
+		Servers:    append([]string(nil), servers...),
+		Version:    version,
+	}
+	lrm.lastBroadcast[filerGroup] = update
+	delete(lrm.pendingTimer, filerGroup)
+	lrm.mu.Unlock()
+
+	glog.V(0).Infof("LockRing: broadcasting ring update for group %q version %d: %v", filerGroup, version, servers)
+
+	if lrm.broadcastFn != nil {
+		lrm.broadcastFn(&master_pb.KeepConnectedResponse{
+			LockRingUpdate: update,
+		})
+	}
+}
+
+// FlushPending fires any pending timer immediately (for testing or shutdown).
+func (lrm *LockRingManager) FlushPending(filerGroup FilerGroupName) {
+	lrm.mu.Lock()
+	if timer, ok := lrm.pendingTimer[filerGroup]; ok {
+		if timer.Stop() {
+			// Timer was pending — we stopped it, so we broadcast now
+			delete(lrm.pendingTimer, filerGroup)
+			lrm.mu.Unlock()
+			lrm.doBroadcast(filerGroup)
+		} else {
+			// Timer already fired, callback is running — let it finish
+			lrm.mu.Unlock()
+		}
+	} else {
+		lrm.mu.Unlock()
+	}
+}
@@ -0,0 +1,229 @@
+package cluster
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLockRingManager_BatchesRapidChanges(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 100 * time.Millisecond
+
+	group := FilerGroupName("default")
+
+	// Add 3 servers in rapid succession
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+	lrm.AddServer(group, "filer3:8888")
+
+	// No broadcast should have happened yet (timer hasn't fired)
+	mu.Lock()
+	assert.Equal(t, 0, len(broadcasts), "should not broadcast before stabilization delay")
+	mu.Unlock()
+
+	// Wait for stabilization
+	time.Sleep(200 * time.Millisecond)
+
+	mu.Lock()
+	require.Equal(t, 1, len(broadcasts), "should batch into a single broadcast")
+	assert.Equal(t, 3, len(broadcasts[0].Servers), "should include all 3 servers")
+	assert.Greater(t, broadcasts[0].Version, int64(0))
+	mu.Unlock()
+}
+
+func TestLockRingManager_DropAndJoinBatched(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 100 * time.Millisecond
+
+	group := FilerGroupName("default")
+
+	// Set up initial state
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+	lrm.AddServer(group, "filer3:8888")
+	lrm.FlushPending(group)
+
+	mu.Lock()
+	broadcasts = nil // reset
+	mu.Unlock()
+
+	// Simulate drop + join in rapid succession
+	lrm.RemoveServer(group, "filer3:8888")
+	lrm.AddServer(group, "filer4:8888")
+
+	// Should not have broadcast yet
+	mu.Lock()
+	assert.Equal(t, 0, len(broadcasts))
+	mu.Unlock()
+
+	// Wait for stabilization
+	time.Sleep(200 * time.Millisecond)
+
+	mu.Lock()
+	require.Equal(t, 1, len(broadcasts), "drop+join should be batched into single broadcast")
+	servers := broadcasts[0].Servers
+	assert.Equal(t, 3, len(servers), "should have filer1, filer2, filer4")
+	// filer3 should be gone, filer4 should be present
+	serverSet := make(map[string]bool)
+	for _, s := range servers {
+		serverSet[s] = true
+	}
+	assert.False(t, serverSet["filer3:8888"], "filer3 should be removed")
+	assert.True(t, serverSet["filer4:8888"], "filer4 should be added")
+	mu.Unlock()
+}
+
+func TestLockRingManager_VersionIncrements(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 50 * time.Millisecond
+
+	group := FilerGroupName("default")
+
+	lrm.AddServer(group, "filer1:8888")
+	time.Sleep(100 * time.Millisecond)
+
+	lrm.AddServer(group, "filer2:8888")
+	time.Sleep(100 * time.Millisecond)
+
+	mu.Lock()
+	require.Equal(t, 2, len(broadcasts))
+	assert.Greater(t, broadcasts[0].Version, int64(0), "version should be positive")
+	assert.Greater(t, broadcasts[1].Version, broadcasts[0].Version, "versions should be monotonically increasing")
+	mu.Unlock()
+}
+
+func TestLockRingManager_FlushPending(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 10 * time.Second // long delay
+
+	group := FilerGroupName("default")
+
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+
+	// Flush immediately
+	lrm.FlushPending(group)
+
+	mu.Lock()
+	require.Equal(t, 1, len(broadcasts))
+	assert.Equal(t, 2, len(broadcasts[0].Servers))
+	mu.Unlock()
+}
+
+func TestLockRingManager_MultipleGroups(t *testing.T) {
+	var mu sync.Mutex
+	broadcastsByGroup := make(map[string][]*master_pb.LockRingUpdate)
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcastsByGroup[resp.LockRingUpdate.FilerGroup] = append(
+				broadcastsByGroup[resp.LockRingUpdate.FilerGroup], resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 50 * time.Millisecond
+
+	lrm.AddServer("group1", "filer1:8888")
+	lrm.AddServer("group2", "filer2:8888")
+
+	time.Sleep(100 * time.Millisecond)
+
+	mu.Lock()
+	assert.Equal(t, 1, len(broadcastsByGroup["group1"]))
+	assert.Equal(t, 1, len(broadcastsByGroup["group2"]))
+	assert.Equal(t, []string{"filer1:8888"}, broadcastsByGroup["group1"][0].Servers)
+	assert.Equal(t, []string{"filer2:8888"}, broadcastsByGroup["group2"][0].Servers)
+	mu.Unlock()
+}
+
+func TestLockRingManager_GetServers(t *testing.T) {
+	lrm := NewLockRingManager(nil)
+
+	group := FilerGroupName("default")
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+
+	servers := lrm.GetServers(group)
+	assert.Equal(t, 2, len(servers))
+
+	// Contains both
+	serverSet := make(map[string]bool)
+	for _, s := range servers {
+		serverSet[s] = true
+	}
+	assert.True(t, serverSet["filer1:8888"])
+	assert.True(t, serverSet["filer2:8888"])
+
+	// Remove one
+	lrm.RemoveServer(group, "filer1:8888")
+	servers = lrm.GetServers(group)
+	assert.Equal(t, 1, len(servers))
+	assert.Equal(t, "filer2:8888", servers[0])
+}
+
+func TestLockRingManager_NoBroadcastWithoutFn(t *testing.T) {
+	// No panic when broadcastFn is nil
+	lrm := NewLockRingManager(nil)
+	lrm.stabilizeDelay = 10 * time.Millisecond
+
+	lrm.AddServer("default", pb.ServerAddress("filer1:8888"))
+	time.Sleep(50 * time.Millisecond) // should not panic
+}
+
+func TestLockRingManager_GetLastUpdateReturnsBroadcastState(t *testing.T) {
+	lrm := NewLockRingManager(nil)
+
+	group := FilerGroupName("default")
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+	lrm.FlushPending(group)
+
+	update := lrm.GetLastUpdate(group)
+	require.NotNil(t, update)
+	assert.Equal(t, "default", update.FilerGroup)
+	assert.ElementsMatch(t, []string{"filer1:8888", "filer2:8888"}, update.Servers)
+	assert.Greater(t, update.Version, int64(0))
+}
@@ -30,6 +30,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/grace"
 )

 var (
@@ -48,6 +49,10 @@ type AdminOptions struct {
 	dataDir          *string
 	icebergPort      *int
 	urlPrefix        *string
+	debug            *bool
+	debugPort        *int
+	cpuProfile       *string
+	memProfile       *string
 }

 func init() {
@@ -64,6 +69,10 @@ func init() {
 	a.readOnlyPassword = cmdAdmin.Flag.String("readOnlyPassword", "", "read-only user password (optional, for view-only access; requires adminPassword to be set)")
 	a.icebergPort = cmdAdmin.Flag.Int("iceberg.port", 8181, "Iceberg REST Catalog port (0 to hide in UI)")
 	a.urlPrefix = cmdAdmin.Flag.String("urlPrefix", "", "URL path prefix when running behind a reverse proxy under a subdirectory (e.g. /seaweedfs)")
+	a.debug = cmdAdmin.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
+	a.debugPort = cmdAdmin.Flag.Int("debug.port", 6060, "http port for debugging")
+	a.cpuProfile = cmdAdmin.Flag.String("cpuprofile", "", "cpu profile output file")
+	a.memProfile = cmdAdmin.Flag.String("memprofile", "", "memory profile output file")
 }

 var cmdAdmin = &Command{
@@ -140,6 +149,16 @@ var cmdAdmin = &Command{
    - All static assets, API endpoints, and navigation links will use the prefix
    - Session cookies are scoped to the prefix path

+  Debugging and Profiling:
+    - Use -debug to start a pprof HTTP server for live profiling (localhost only)
+    - Set -debug.port to choose the pprof port (default 6060)
+    - Profiles are accessible at http://127.0.0.1:<debug.port>/debug/pprof/
+    - Use -cpuprofile and -memprofile to write profiles to files on shutdown
+    - WARNING: -debug exposes runtime internals; use only in trusted environments
+    - Examples:
+      weed admin -debug -debug.port=6060 -master="localhost:9333"
+      weed admin -cpuprofile=cpu.prof -memprofile=mem.prof -master="localhost:9333"
+
  Configuration File:
    - The security.toml file is read from ".", "$HOME/.seaweedfs/",
      "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order
@@ -149,6 +168,12 @@ var cmdAdmin = &Command{
 }

 func runAdmin(cmd *Command, args []string) bool {
+	if *a.debug {
+		grace.StartDebugServer(*a.debugPort)
+	}
+
+	grace.SetupProfiling(*a.cpuProfile, *a.memProfile)
+
 	// Load security configuration
 	util.LoadSecurityConfiguration()

@@ -511,11 +536,6 @@ func recoveryMiddleware(next http.Handler) http.Handler {
 	})
 }

-// GetAdminOptions returns the admin command options for testing
-func GetAdminOptions() *AdminOptions {
-	return &AdminOptions{}
-}
-
 // loadOrGenerateSessionKeys loads or creates authentication/encryption keys for session cookies.
 func loadOrGenerateSessionKeys(dataDir string) ([]byte, []byte, error) {
 	const keyLen = 32
@@ -132,16 +132,3 @@ func fetchContent(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption
 	content, e = io.ReadAll(rc.Body)
 	return
 }
-
-func WriteFile(filename string, data []byte, perm os.FileMode) error {
-	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
-	if err != nil {
-		return err
-	}
-	n, err := f.Write(data)
-	f.Close()
-	if err == nil && n < len(data) {
-		err = io.ErrShortWrite
-	}
-	return err
-}
@@ -81,6 +81,7 @@ type FilerOptions struct {
 	exposeDirectoryData       *bool
 	tusBasePath               *string
 	certProvider              certprovider.Provider
+	s3ConfigFile              *string // optional path to static S3 identity config
 }

 func init() {
@@ -146,6 +147,7 @@ func init() {
 	filerS3Options.iamReadOnly = cmdFiler.Flag.Bool("s3.iam.readOnly", true, "disable IAM write operations on this server")
 	filerS3Options.portIceberg = cmdFiler.Flag.Int("s3.port.iceberg", 8181, "Iceberg REST Catalog server listen port (0 to disable)")
 	filerS3Options.externalUrl = cmdFiler.Flag.String("s3.externalUrl", "", "the external URL clients use to connect (e.g. https://api.example.com:9000). Used for S3 signature verification behind a reverse proxy. Falls back to S3_EXTERNAL_URL env var.")
+	filerS3Options.defaultFileMode = cmdFiler.Flag.String("s3.defaultFileMode", "", "default file mode for S3 uploaded objects, e.g. 0660, 0644, 0666")

 	// start webdav on filer
 	filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
@@ -342,6 +344,15 @@ func (fo *FilerOptions) startFiler() {
 		glog.V(0).Infof("Initialized credential manager: %s", credentialManager.GetStoreName())
 	}

+	// Load static S3 identities from config file if specified
+	if fo.s3ConfigFile != nil && *fo.s3ConfigFile != "" {
+		if credentialManager != nil {
+			if err := credentialManager.LoadS3ConfigFile(*fo.s3ConfigFile); err != nil {
+				glog.Warningf("Failed to load S3 config file for static identities: %v", err)
+			}
+		}
+	}
+
 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
 		Masters:                   fo.masters,
 		FilerGroup:                *fo.filerGroup,
@@ -421,6 +432,7 @@ func (fo *FilerOptions) startFiler() {
 		go grpcS.Serve(grpcLocalL)
 	}
 	go grpcS.Serve(grpcL)
+	pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)

 	if runtime.GOOS != "windows" {
 		localSocket := *fo.localSocket
@@ -164,7 +164,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour

 	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification
-		if message.NewEntry != nil {
+		if metadataEventUpdatesDirectory(resp, filer.DirectoryEtcRemote) {
 			// update
 			if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
 				newMappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
@@ -180,8 +180,11 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				}
 				option.remoteConfs[conf.Name] = conf
 			}
-		} else if message.OldEntry != nil {
+		} else if metadataEventRemovesFromDirectory(resp, filer.DirectoryEtcRemote) {
 			// deletion
+			if message.OldEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
+				option.mappings = &remote_pb.RemoteStorageMapping{}
+			}
 			if strings.HasSuffix(message.OldEntry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
 				conf := &remote_pb.RemoteConf{}
 				if err := proto.Unmarshal(message.OldEntry.Content, conf); err != nil {
@@ -196,7 +199,8 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour

 	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification
-		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
+		sourceInEtcRemote, targetInEtcRemote := metadataEventDirectoryMembership(resp, filer.DirectoryEtcRemote)
+		if sourceInEtcRemote || targetInEtcRemote {
 			return handleEtcRemoteChanges(resp)
 		}

@@ -92,34 +92,38 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem

 	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification
-		if message.NewEntry == nil {
-			return nil
-		}
-		if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
-			mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
-			if readErr != nil {
-				return fmt.Errorf("unmarshal mappings: %w", readErr)
+		if metadataEventUpdatesDirectory(resp, filer.DirectoryEtcRemote) {
+			if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
+				mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
+				if readErr != nil {
+					return fmt.Errorf("unmarshal mappings: %w", readErr)
+				}
+				if remoteLoc, found := mappings.Mappings[mountedDir]; found {
+					if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
+						glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
+					}
+				} else {
+					glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
+					os.Exit(0)
+				}
 			}
-			if remoteLoc, found := mappings.Mappings[mountedDir]; found {
-				if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
-					glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
+			if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
+				conf := &remote_pb.RemoteConf{}
+				if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
+					return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
+				}
+				remoteStorage = conf
+				if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil {
+					client = newClient
+				} else {
+					return err
 				}
-			} else {
-				glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
-				os.Exit(0)
 			}
 		}
-		if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
-			conf := &remote_pb.RemoteConf{}
-			if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
-				return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
-			}
-			remoteStorage = conf
-			if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil {
-				client = newClient
-			} else {
-				return err
-			}
+		if metadataEventRemovesFromDirectory(resp, filer.DirectoryEtcRemote) &&
+			message.OldEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
+			glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
+			os.Exit(0)
 		}

 		return nil
@@ -127,7 +131,8 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem

 	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification
-		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
+		sourceInEtcRemote, targetInEtcRemote := metadataEventDirectoryMembership(resp, filer.DirectoryEtcRemote)
+		if sourceInEtcRemote || targetInEtcRemote {
 			return handleEtcRemoteChanges(resp)
 		}

@@ -26,37 +26,39 @@ import (
 )

 type SyncOptions struct {
-	isActivePassive *bool
-	filerA          *string
-	filerB          *string
-	aPath           *string
-	aExcludePaths   *string
-	bPath           *string
-	bExcludePaths   *string
-	aReplication    *string
-	bReplication    *string
-	aCollection     *string
-	bCollection     *string
-	aTtlSec         *int
-	bTtlSec         *int
-	aDiskType       *string
-	bDiskType       *string
-	aDebug          *bool
-	bDebug          *bool
-	aFromTsMs       *int64
-	bFromTsMs       *int64
-	aProxyByFiler   *bool
-	bProxyByFiler   *bool
-	metricsHttpIp   *string
-	metricsHttpPort *int
+	isActivePassive  *bool
+	filerA           *string
+	filerB           *string
+	aPath            *string
+	aExcludePaths    *string
+	bPath            *string
+	bExcludePaths    *string
+	aReplication     *string
+	bReplication     *string
+	aCollection      *string
+	bCollection      *string
+	aTtlSec          *int
+	bTtlSec          *int
+	aDiskType        *string
+	bDiskType        *string
+	aDebug           *bool
+	bDebug           *bool
+	aFromTsMs        *int64
+	bFromTsMs        *int64
+	aProxyByFiler    *bool
+	bProxyByFiler    *bool
+	metricsHttpIp    *string
+	metricsHttpPort  *int
 	concurrency      *int
 	chunkConcurrency *int
 	aDoDeleteFiles   *bool
-	bDoDeleteFiles  *bool
-	clientId        int32
-	clientEpoch     atomic.Int32
-	debug           *bool
-	debugPort       *int
+	bDoDeleteFiles   *bool
+	aSecurity        *string
+	bSecurity        *string
+	clientId         int32
+	clientEpoch      atomic.Int32
+	debug            *bool
+	debugPort        *int
 }

 const (
@@ -113,6 +115,8 @@ func init() {
 	syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port")
 	syncOptions.aDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("a.doDeleteFiles", true, "delete and update files when synchronizing on filer A")
 	syncOptions.bDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("b.doDeleteFiles", true, "delete and update files when synchronizing on filer B")
+	syncOptions.aSecurity = cmdFilerSynchronize.Flag.String("a.security", "", "security.toml file for filer A when clusters use different certificates")
+	syncOptions.bSecurity = cmdFilerSynchronize.Flag.String("b.security", "", "security.toml file for filer B when clusters use different certificates")
 	syncOptions.debug = cmdFilerSynchronize.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
 	syncOptions.debugPort = cmdFilerSynchronize.Flag.Int("debug.port", 6060, "http port for debugging")
 	syncOptions.clientId = util.RandomInt32()
@@ -144,6 +148,22 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	util.LoadSecurityConfiguration()
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")

+	// per-filer TLS when clusters use different certificates
+	grpcDialOptionA := grpcDialOption
+	grpcDialOptionB := grpcDialOption
+	if *syncOptions.aSecurity != "" {
+		var err error
+		if grpcDialOptionA, err = security.LoadClientTLSFromFile(*syncOptions.aSecurity, "grpc.client"); err != nil {
+			glog.Fatalf("load security config for filer A: %v", err)
+		}
+	}
+	if *syncOptions.bSecurity != "" {
+		var err error
+		if grpcDialOptionB, err = security.LoadClientTLSFromFile(*syncOptions.bSecurity, "grpc.client"); err != nil {
+			glog.Fatalf("load security config for filer B: %v", err)
+		}
+	}
+
 	grace.SetupProfiling(*syncCpuProfile, *syncMemProfile)

 	filerA := pb.ServerAddress(*syncOptions.filerA)
@@ -153,13 +173,13 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	go statsCollect.StartMetricsServer(*syncOptions.metricsHttpIp, *syncOptions.metricsHttpPort)

 	// read a filer signature
-	aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOption, filerA)
+	aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOptionA, filerA)
 	if aFilerErr != nil {
 		glog.Errorf("get filer 'a' signature %d error from %s to %s: %v", aFilerSignature, *syncOptions.filerA, *syncOptions.filerB, aFilerErr)
 		return true
 	}
 	// read b filer signature
-	bFilerSignature, bFilerErr := replication.ReadFilerSignature(grpcDialOption, filerB)
+	bFilerSignature, bFilerErr := replication.ReadFilerSignature(grpcDialOptionB, filerB)
 	if bFilerErr != nil {
 		glog.Errorf("get filer 'b' signature %d error from %s to %s: %v", bFilerSignature, *syncOptions.filerA, *syncOptions.filerB, bFilerErr)
 		return true
@@ -189,9 +209,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	go func() {
 		// a->b
 		// set synchronization start timestamp to offset
-		initOffsetError := initOffsetFromTsMs(grpcDialOption, filerB, aFilerSignature, *syncOptions.bFromTsMs, getSignaturePrefixByPath(*syncOptions.aPath))
+		initOffsetError := initOffsetFromTsMs(grpcDialOptionB, filerB, aFilerSignature, *syncOptions.aFromTsMs, getSignaturePrefixByPath(*syncOptions.aPath))
 		if initOffsetError != nil {
-			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.bFromTsMs, *syncOptions.filerA, *syncOptions.filerB, initOffsetError)
+			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.aFromTsMs, *syncOptions.filerA, *syncOptions.filerB, initOffsetError)
 			os.Exit(2)
 		}
 		for {
@@ -199,11 +219,12 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 			err := doSubscribeFilerMetaChanges(
 				syncOptions.clientId,
 				syncOptions.clientEpoch.Load(),
-				grpcDialOption,
+				grpcDialOptionA,
 				filerA,
 				*syncOptions.aPath,
 				util.StringSplit(*syncOptions.aExcludePaths, ","),
 				*syncOptions.aProxyByFiler,
+				grpcDialOptionB,
 				filerB,
 				*syncOptions.bPath,
 				*syncOptions.bReplication,
@@ -228,9 +249,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	if !*syncOptions.isActivePassive {
 		// b->a
 		// set synchronization start timestamp to offset
-		initOffsetError := initOffsetFromTsMs(grpcDialOption, filerA, bFilerSignature, *syncOptions.aFromTsMs, getSignaturePrefixByPath(*syncOptions.bPath))
+		initOffsetError := initOffsetFromTsMs(grpcDialOptionA, filerA, bFilerSignature, *syncOptions.bFromTsMs, getSignaturePrefixByPath(*syncOptions.bPath))
 		if initOffsetError != nil {
-			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.aFromTsMs, *syncOptions.filerB, *syncOptions.filerA, initOffsetError)
+			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.bFromTsMs, *syncOptions.filerB, *syncOptions.filerA, initOffsetError)
 			os.Exit(2)
 		}
 		go func() {
@@ -239,11 +260,12 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 				err := doSubscribeFilerMetaChanges(
 					syncOptions.clientId,
 					syncOptions.clientEpoch.Load(),
-					grpcDialOption,
+					grpcDialOptionB,
 					filerB,
 					*syncOptions.bPath,
 					util.StringSplit(*syncOptions.bExcludePaths, ","),
 					*syncOptions.bProxyByFiler,
+					grpcDialOptionA,
 					filerA,
 					*syncOptions.aPath,
 					*syncOptions.aReplication,
@@ -285,12 +307,12 @@ func initOffsetFromTsMs(grpcDialOption grpc.DialOption, targetFiler pb.ServerAdd
 	return nil
 }

-func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceExcludePaths []string, sourceReadChunkFromFiler bool, targetFiler pb.ServerAddress, targetPath string,
+func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceExcludePaths []string, sourceReadChunkFromFiler bool, targetGrpcDialOption grpc.DialOption, targetFiler pb.ServerAddress, targetPath string,
 	replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool, concurrency int, chunkConcurrency int, doDeleteFiles bool, sourceFilerSignature int32, targetFilerSignature int32, statePtr *atomic.Pointer[syncState]) error {

 	// if first time, start from now
 	// if has previously synced, resume from that point of time
-	sourceFilerOffsetTsNs, err := getOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature)
+	sourceFilerOffsetTsNs, err := getOffset(targetGrpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature)
 	if err != nil {
 		return err
 	}
@@ -300,8 +322,9 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 	// create filer sink
 	filerSource := &source.FilerSource{}
 	filerSource.DoInitialize(sourceFiler.ToHttpAddress(), sourceFiler.ToGrpcAddress(), sourcePath, sourceReadChunkFromFiler)
+	filerSource.SetGrpcDialOption(sourceGrpcDialOption)
 	filerSink := &filersink.FilerSink{}
-	filerSink.DoInitialize(targetFiler.ToHttpAddress(), targetFiler.ToGrpcAddress(), targetPath, replicationStr, collection, ttlSec, diskType, grpcDialOption, sinkWriteChunkByFiler)
+	filerSink.DoInitialize(targetFiler.ToHttpAddress(), targetFiler.ToGrpcAddress(), targetPath, replicationStr, collection, ttlSec, diskType, targetGrpcDialOption, sinkWriteChunkByFiler)
 	filerSink.SetChunkConcurrency(chunkConcurrency)
 	filerSink.SetSourceFiler(filerSource)

@@ -328,7 +351,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 	if statePtr != nil {
 		statePtr.Store(&syncState{
 			processor:            processor,
-			grpcDialOption:       grpcDialOption,
+			grpcDialOption:       targetGrpcDialOption,
 			targetFiler:          targetFiler,
 			sourcePath:           sourcePath,
 			sourceFilerSignature: sourceFilerSignature,
@@ -336,6 +359,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 	}

 	var lastLogTsNs = time.Now().UnixNano()
+	var lastProgressedTsNs int64
 	var clientName = fmt.Sprintf("syncFrom_%s_To_%s", string(sourceFiler), string(targetFiler))
 	processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
 		processor.AddSyncJob(resp)
@@ -349,9 +373,21 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 		now := time.Now().UnixNano()
 		glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
 		lastLogTsNs = now
+		if offsetTsNs == lastProgressedTsNs {
+			for _, t := range filerSink.ActiveTransfers() {
+				if t.LastErr != "" {
+					glog.V(0).Infof("  %s %s: %d bytes received, %s, last error: %s",
+						t.ChunkFileId, t.Path, t.BytesReceived, t.Status, t.LastErr)
+				} else {
+					glog.V(0).Infof("  %s %s: %d bytes received, %s",
+						t.ChunkFileId, t.Path, t.BytesReceived, t.Status)
+				}
+			}
+		}
+		lastProgressedTsNs = offsetTsNs
 		// collect synchronous offset
 		statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(offsetTsNs))
-		return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
+		return setOffset(targetGrpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
 	})

 	prefix := sourcePath
@@ -372,7 +408,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 		EventErrorType:         pb.RetryForeverOnError,
 	}

-	return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
+	return pb.FollowMetadata(sourceFiler, sourceGrpcDialOption, metadataFollowOption, processEventFnWithOffset)

 }

@@ -445,12 +481,17 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification

+		// Derive the target (new-side) directory once. MetadataEventTargetDirectory
+		// returns NewParentPath when set, falling back to resp.Directory for
+		// delete events or legacy events with an empty NewParentPath.
+		targetDir := filer_pb.MetadataEventTargetDirectory(resp)
+
 		var sourceOldKey, sourceNewKey util.FullPath
 		if message.OldEntry != nil {
 			sourceOldKey = util.FullPath(resp.Directory).Child(message.OldEntry.Name)
 		}
 		if message.NewEntry != nil {
-			sourceNewKey = util.FullPath(message.NewParentPath).Child(message.NewEntry.Name)
+			sourceNewKey = util.FullPath(targetDir).Child(message.NewEntry.Name)
 		}

 		if debug {
@@ -461,19 +502,24 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			return nil
 		}

-		if !strings.HasPrefix(resp.Directory+"/", sourcePath) {
+		// For rename events the key/directory is the old (source) path.
+		// Check both old and new directories so cross-boundary renames
+		// are not silently dropped. The downstream old/new key handling
+		// (lines below) already converts these to create or delete.
+		oldDirExcluded := matchesExcludePath(resp.Directory, excludePaths)
+		newDirExcluded := matchesExcludePath(targetDir, excludePaths)
+		oldDirInScope := util.IsEqualOrUnder(resp.Directory, sourcePath) && !oldDirExcluded
+		newDirInScope := message.NewEntry != nil &&
+			util.IsEqualOrUnder(targetDir, sourcePath) &&
+			!newDirExcluded
+		if !oldDirInScope && !newDirInScope {
 			return nil
 		}
-		for _, excludePath := range excludePaths {
-			if strings.HasPrefix(resp.Directory+"/", excludePath) {
-				return nil
-			}
-		}
 		// Compute per-side exclusion so that rename events crossing an
 		// exclude boundary are handled as delete + create rather than
 		// being entirely skipped.
-		oldExcluded := isEntryExcluded(resp.Directory, message.OldEntry, reExcludeFileName, excludeFileNames, excludePathPatterns)
-		newExcluded := isEntryExcluded(message.NewParentPath, message.NewEntry, reExcludeFileName, excludeFileNames, excludePathPatterns)
+		oldExcluded := oldDirExcluded || isEntryExcluded(resp.Directory, message.OldEntry, reExcludeFileName, excludeFileNames, excludePathPatterns)
+		newExcluded := newDirExcluded || isEntryExcluded(targetDir, message.NewEntry, reExcludeFileName, excludeFileNames, excludePathPatterns)

 		if oldExcluded && newExcluded {
 			return nil
@@ -495,7 +541,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			if !doDeleteFiles {
 				return nil
 			}
-			if !strings.HasPrefix(string(sourceOldKey), sourcePath) {
+			if !util.IsEqualOrUnder(string(sourceOldKey), sourcePath) {
 				return nil
 			}
 			key := buildKey(dataSink, message, targetPath, sourceOldKey, sourcePath)
@@ -504,7 +550,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str

 		// handle new entries
 		if filer_pb.IsCreate(resp) {
-			if !strings.HasPrefix(string(sourceNewKey), sourcePath) {
+			if !util.IsEqualOrUnder(string(sourceNewKey), sourcePath) {
 				return nil
 			}
 			key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
@@ -521,18 +567,19 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 		}

 		// handle updates
-		if strings.HasPrefix(string(sourceOldKey), sourcePath) {
+		if util.IsEqualOrUnder(string(sourceOldKey), sourcePath) {
 			// old key is in the watched directory
-			if strings.HasPrefix(string(sourceNewKey), sourcePath) {
+			if util.IsEqualOrUnder(string(sourceNewKey), sourcePath) {
 				// new key is also in the watched directory
 				if doDeleteFiles {
 					oldKey := util.Join(targetPath, string(sourceOldKey)[len(sourcePath):])
+					var sinkNewParentPath string
 					if strings.HasSuffix(sourcePath, "/") {
-						message.NewParentPath = util.Join(targetPath, message.NewParentPath[len(sourcePath)-1:])
+						sinkNewParentPath = util.Join(targetPath, targetDir[len(sourcePath)-1:])
 					} else {
-						message.NewParentPath = util.Join(targetPath, message.NewParentPath[len(sourcePath):])
+						sinkNewParentPath = util.Join(targetPath, targetDir[len(sourcePath):])
 					}
-					foundExisting, err := dataSink.UpdateEntry(string(oldKey), message.OldEntry, message.NewParentPath, message.NewEntry, message.DeleteChunks, message.Signatures)
+					foundExisting, err := dataSink.UpdateEntry(string(oldKey), message.OldEntry, sinkNewParentPath, message.NewEntry, message.DeleteChunks, message.Signatures)
 					if foundExisting {
 						return err
 					}
@@ -559,7 +606,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			}
 		} else {
 			// old key is outside the watched directory
-			if strings.HasPrefix(string(sourceNewKey), sourcePath) {
+			if util.IsEqualOrUnder(string(sourceNewKey), sourcePath) {
 				// new key is in the watched directory
 				key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
 				if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil {
@@ -623,6 +670,15 @@ func isEntryExcluded(dir string, entry *filer_pb.Entry, reExcludeFileName *regex
 	return false
 }

+func matchesExcludePath(dir string, excludePaths []string) bool {
+	for _, excludePath := range excludePaths {
+		if util.IsEqualOrUnder(dir, excludePath) {
+			return true
+		}
+	}
+	return false
+}
+
 // compileExcludePattern compiles a regexp pattern string, returning nil if empty.
 func compileExcludePattern(pattern string, label string) (*regexp.Regexp, error) {
 	if pattern == "" {
@@ -0,0 +1,121 @@
+package command
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var _ sink.ReplicationSink = (*recordingSyncSink)(nil)
+
+type recordingSyncSink struct {
+	deleteKeys []string
+	createKeys []string
+	updateKeys []string
+}
+
+func (s *recordingSyncSink) GetName() string { return "recording" }
+func (s *recordingSyncSink) Initialize(util.Configuration, string) error {
+	return nil
+}
+func (s *recordingSyncSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
+	s.deleteKeys = append(s.deleteKeys, key)
+	return nil
+}
+func (s *recordingSyncSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
+	s.createKeys = append(s.createKeys, key)
+	return nil
+}
+func (s *recordingSyncSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (bool, error) {
+	s.updateKeys = append(s.updateKeys, key)
+	return true, nil
+}
+func (s *recordingSyncSink) GetSinkToDirectory() string         { return "/dest" }
+func (s *recordingSyncSink) SetSourceFiler(*source.FilerSource) {}
+func (s *recordingSyncSink) IsIncremental() bool                { return false }
+
+func TestPathIsEqualOrUnderUsesDirectoryBoundaries(t *testing.T) {
+	tests := []struct {
+		name      string
+		candidate string
+		other     string
+		expected  bool
+	}{
+		{name: "equal", candidate: "/foo", other: "/foo", expected: true},
+		{name: "descendant", candidate: "/foo/bar", other: "/foo", expected: true},
+		{name: "sibling prefix", candidate: "/foobar/bar", other: "/foo", expected: false},
+		{name: "root", candidate: "/foo/bar", other: "/", expected: true},
+		{name: "empty", candidate: "", other: "/foo", expected: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := util.IsEqualOrUnder(tt.candidate, tt.other); got != tt.expected {
+				t.Fatalf("IsEqualOrUnder(%q, %q) = %v, want %v", tt.candidate, tt.other, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestMatchesExcludePathUsesDirectoryBoundaries(t *testing.T) {
+	if !matchesExcludePath("/tmp", []string{"/tmp"}) {
+		t.Fatal("expected exact directory match to be excluded")
+	}
+	if !matchesExcludePath("/tmp/sub", []string{"/tmp"}) {
+		t.Fatal("expected descendant directory to be excluded")
+	}
+	if matchesExcludePath("/tmp2/sub", []string{"/tmp"}) {
+		t.Fatal("did not expect sibling directory to be excluded")
+	}
+}
+
+func TestGenProcessFunctionRenameToSiblingPrefixBecomesDelete(t *testing.T) {
+	dataSink := &recordingSyncSink{}
+	processFn := genProcessFunction("/foo", "/dest", nil, nil, nil, nil, dataSink, true, false)
+
+	err := processFn(&filer_pb.SubscribeMetadataResponse{
+		Directory: "/foo/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: "file.txt"},
+			NewEntry:      &filer_pb.Entry{Name: "file.txt"},
+			NewParentPath: "/foobar/dir",
+		},
+	})
+	if err != nil {
+		t.Fatalf("processFn rename to sibling prefix: %v", err)
+	}
+
+	if len(dataSink.deleteKeys) != 1 || dataSink.deleteKeys[0] != "/dest/dir/file.txt" {
+		t.Fatalf("delete keys = %v, want [/dest/dir/file.txt]", dataSink.deleteKeys)
+	}
+	if len(dataSink.createKeys) != 0 || len(dataSink.updateKeys) != 0 {
+		t.Fatalf("unexpected create/update calls: creates=%v updates=%v", dataSink.createKeys, dataSink.updateKeys)
+	}
+}
+
+func TestGenProcessFunctionRenameFromExcludedDirBecomesCreate(t *testing.T) {
+	dataSink := &recordingSyncSink{}
+	processFn := genProcessFunction("/foo", "/dest", []string{"/foo/excluded"}, nil, nil, nil, dataSink, true, false)
+
+	err := processFn(&filer_pb.SubscribeMetadataResponse{
+		Directory: "/foo/excluded",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: "file.txt"},
+			NewEntry:      &filer_pb.Entry{Name: "file.txt"},
+			NewParentPath: "/foo/live",
+		},
+	})
+	if err != nil {
+		t.Fatalf("processFn rename from excluded dir: %v", err)
+	}
+
+	if len(dataSink.createKeys) != 1 || dataSink.createKeys[0] != "/dest/live/file.txt" {
+		t.Fatalf("create keys = %v, want [/dest/live/file.txt]", dataSink.createKeys)
+	}
+	if len(dataSink.deleteKeys) != 0 || len(dataSink.updateKeys) != 0 {
+		t.Fatalf("unexpected delete/update calls: deletes=%v updates=%v", dataSink.deleteKeys, dataSink.updateKeys)
+	}
+}
@@ -254,6 +254,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		go grpcS.Serve(grpcLocalL)
 	}
 	go grpcS.Serve(grpcL)
+	pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)

 	// For multi-master mode with non-Hashicorp raft, wait and check if we should join
 	if !*masterOption.raftHashicorp && !isSingleMaster {
@@ -0,0 +1,36 @@
+package command
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func metadataEventDirectoryMembership(resp *filer_pb.SubscribeMetadataResponse, dir string) (sourceInDir, targetInDir bool) {
+	if resp == nil || resp.EventNotification == nil {
+		return false, false
+	}
+
+	sourceInDir = util.IsEqualOrUnder(resp.Directory, dir)
+	targetInDir = resp.EventNotification.NewEntry != nil &&
+		util.IsEqualOrUnder(filer_pb.MetadataEventTargetDirectory(resp), dir)
+
+	return sourceInDir, targetInDir
+}
+
+func metadataEventUpdatesDirectory(resp *filer_pb.SubscribeMetadataResponse, dir string) bool {
+	if resp == nil || resp.EventNotification == nil || resp.EventNotification.NewEntry == nil {
+		return false
+	}
+
+	_, targetInDir := metadataEventDirectoryMembership(resp, dir)
+	return targetInDir
+}
+
+func metadataEventRemovesFromDirectory(resp *filer_pb.SubscribeMetadataResponse, dir string) bool {
+	if resp == nil || resp.EventNotification == nil || resp.EventNotification.OldEntry == nil {
+		return false
+	}
+
+	sourceInDir, targetInDir := metadataEventDirectoryMembership(resp, dir)
+	return sourceInDir && !targetInDir
+}
--- a/Show More
+++ b/Show More
				`@@ -0,0 +1 @@`
				`{"sessionId":"d6574c47-eafc-4a94-9dce-f9ffea22b53c","pid":10111,"acquiredAt":1775248373916}`