4.18

* filer.sync: support per-cluster mTLS with -a.security and -b.security flags

When syncing between two clusters that use different certificate authorities,
a single security.toml cannot authenticate to both. Add -a.security and
-b.security flags so each filer can use its own security.toml for TLS.

Closes #8481

* security: fatal on failure to read explicitly provided security config

When -a.security or -b.security is specified, falling back to insecure
credentials on read error would silently bypass mTLS. Fatal instead.

* fix(filer.sync): use source filer's fromTsMs flag in initOffsetFromTsMs

A→B was using bFromTsMs and B→A was using aFromTsMs — these were
swapped. Each path should seed the target's offset with the source
filer's starting timestamp.

* security: return error from LoadClientTLSFromFile, resolve relative PEM paths

Change LoadClientTLSFromFile to return (grpc.DialOption, error) so
callers can handle failures explicitly instead of a silent insecure
fallback. Resolve relative PEM paths (grpc.ca, grpc.client.cert,
grpc.client.key) against the config file's directory.

.github/workflows/container_latest.yml

@@ -23,6 +23,11 @@ on:
           - all
           - standard
           - large_disk
+      publish:
+        description: 'Publish images and manifests'
+        required: true
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -33,6 +38,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       variants: ${{ steps.set-variants.outputs.variants }}
+      publish: ${{ steps.set-publish.outputs.publish }}
     steps:
       - name: Select variants for this run
         id: set-variants
@@ -43,6 +49,14 @@ jobs:
             variants='["standard","large_disk"]'
           fi
           echo "variants=$variants" >> "$GITHUB_OUTPUT"
+      - name: Select publish mode
+        id: set-publish
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
+          else
+            echo "publish=true" >> "$GITHUB_OUTPUT"
+          fi
 
   build:
     needs: [setup]
@@ -112,13 +126,13 @@ jobs:
           buildkitd-flags: "--debug"
           buildkitd-config: /tmp/buildkitd.toml
       - name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
+        if: needs.setup.outputs.publish == 'true'
         uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Login to GHCR
-        if: github.event_name != 'pull_request'
+        if: needs.setup.outputs.publish == 'true'
         uses: docker/login-action@v4
         with:
           registry: ghcr.io
@@ -130,7 +144,7 @@ jobs:
           DOCKER_BUILDKIT: 1
         with:
           context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ needs.setup.outputs.publish == 'true' }}
           file: ./docker/Dockerfile.go_build
           platforms: linux/${{ matrix.platform }}
           # Push to GHCR only during build to avoid Docker Hub rate limits
@@ -166,33 +180,112 @@ jobs:
             echo "tag_suffix=" >> $GITHUB_OUTPUT
           fi
       - name: Login to GHCR
+        if: needs.setup.outputs.publish == 'true'
         uses: docker/login-action@v4
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
-      - name: Run Trivy vulnerability scanner
-        # Pin to SHA — mutable tags were compromised (GHSA-69fq-xp46-6x23)
+      - name: Checkout for local scan build
+        if: needs.setup.outputs.publish != 'true'
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
+      - name: Create BuildKit config for local scan build
+        if: needs.setup.outputs.publish != 'true'
+        run: |
+          cat > /tmp/buildkitd.toml <<EOF
+          [registry."docker.io"]
+            mirrors = ["https://mirror.gcr.io"]
+          EOF
+      - name: Set up Docker Buildx for local scan build
+        if: needs.setup.outputs.publish != 'true'
+        uses: docker/setup-buildx-action@v4
+        with:
+          buildkitd-flags: "--debug"
+          buildkitd-config: /tmp/buildkitd.toml
+      - name: Build local scan image tarball
+        if: needs.setup.outputs.publish != 'true'
+        uses: docker/build-push-action@v7
+        env:
+          DOCKER_BUILDKIT: 1
+        with:
+          context: ./docker
+          file: ./docker/Dockerfile.go_build
+          platforms: linux/amd64
+          outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1
+            BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
+            ${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
+      - name: Trivy report (published image)
+        if: needs.setup.outputs.publish == 'true'
+        # Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
         uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
         with:
-          # Scan amd64 only — OS packages are identical across architectures
+          scan-type: image
+          # Scan amd64 only - OS packages are identical across architectures
           # since they all use the same alpine base, so a single-arch scan
           # provides sufficient coverage without multiplying CI time.
           image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
+          scanners: vuln
+          vuln-type: os,library
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          limit-severities-for-sarif: true
           format: sarif
           output: trivy-results.sarif
+          exit-code: '0'
+      - name: Trivy report (local tarball)
+        if: needs.setup.outputs.publish != 'true'
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
+          scanners: vuln
+          vuln-type: os,library
           severity: HIGH,CRITICAL
-          exit-code: '1'
+          ignore-unfixed: true
+          limit-severities-for-sarif: true
+          format: sarif
+          output: trivy-results.sarif
+          exit-code: '0'
       - name: Upload Trivy scan results to GitHub Security
-        uses: github/codeql-action/upload-sarif@v3
+        uses: github/codeql-action/upload-sarif@v4
         if: always()
         with:
           sarif_file: trivy-results.sarif
+      - name: Trivy gate (published image)
+        if: needs.setup.outputs.publish == 'true'
+        # Gate only on fixable high/critical vulnerabilities. Non-fixable
+        # findings are still visible in the SARIF upload above.
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          scan-type: image
+          image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
+          scanners: vuln
+          vuln-type: os,library
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          format: table
+          exit-code: '1'
+          skip-setup-trivy: true
+      - name: Trivy gate (local tarball)
+        if: needs.setup.outputs.publish != 'true'
+        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        with:
+          input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
+          scanners: vuln
+          vuln-type: os,library
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+          format: table
+          exit-code: '1'
+          skip-setup-trivy: true
 
   create-manifest:
     runs-on: ubuntu-latest
     needs: [setup, build, trivy-scan]
-    if: github.event_name != 'pull_request'
+    if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
     strategy:
       matrix:
         variant: ${{ fromJSON(needs.setup.outputs.variants) }}

.github/workflows/helm_ci.yml

@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v4
+        uses: azure/setup-helm@v5
         with:
           version: v3.18.4
 

.github/workflows/rust-volume-server-tests.yml

@@ -43,7 +43,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry
@@ -80,7 +80,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry
@@ -156,7 +156,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry

.github/workflows/rust_binaries_dev.yml

@@ -47,7 +47,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry
@@ -118,7 +118,7 @@ jobs:
           targets: ${{ matrix.target }}
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry

.github/workflows/rust_binaries_release.yml

@@ -43,7 +43,7 @@ jobs:
           echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry
@@ -112,7 +112,7 @@ jobs:
           targets: ${{ matrix.target }}
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry
@@ -170,7 +170,7 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Cache cargo registry and target
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cargo/registry

docker/Dockerfile.go_build

@@ -16,15 +16,16 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
   && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
   && CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
 
-# Rust volume server builder (amd64/arm64 only)
-FROM rust:1-alpine as rust_builder
+# Rust volume server builder. Alpine packages avoid depending on the
+# upstream rust:alpine manifest list, which no longer includes linux/386.
+FROM alpine:3.23 as rust_builder
 ARG TARGETARCH
-RUN apk add musl-dev protobuf-dev git
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
-COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/proto /build/proto
+COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
 WORKDIR /build/seaweed-volume
 ARG TAGS
 RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
+      apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
       if [ "$TAGS" = "5BytesOffset" ]; then \
         cargo build --release; \
       else \
@@ -49,7 +50,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
 # To disable: docker run -e GODEBUG=fips140=off ...
 
 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse curl su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse curl su-exec && \
     addgroup -g 1000 seaweed && \
     adduser -D -u 1000 -G seaweed seaweed
 
@@ -79,9 +81,5 @@ RUN mkdir -p /data/filerldb2 && \
 VOLUME /data
 WORKDIR /data
 
-# Run as non-root by default (satisfies security scanners).
-# Use `docker run --user root` if you need the entrypoint to fix
-# /data volume ownership before dropping privileges.
-USER seaweed
-
+# Entrypoint will handle permission fixes and user switching
 ENTRYPOINT ["/entrypoint.sh"]

docker/Dockerfile.local

@@ -7,7 +7,8 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
 COPY ./entrypoint.sh /entrypoint.sh
 
 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse curl su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse curl su-exec && \
     addgroup -g 1000 seaweed && \
     adduser -D -u 1000 -G seaweed seaweed
 
@@ -37,9 +38,5 @@ RUN mkdir -p /data/filerldb2 && \
 VOLUME /data
 WORKDIR /data
 
-# Run as non-root by default (satisfies security scanners).
-# Use `docker run --user root` if you need the entrypoint to fix
-# /data volume ownership before dropping privileges.
-USER seaweed
-
+# Entrypoint will handle permission fixes and user switching
 ENTRYPOINT ["/entrypoint.sh"]

docker/Dockerfile.rocksdb_large

@@ -34,7 +34,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
 
 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse snappy gflags curl su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse snappy gflags curl su-exec && \
     addgroup -g 1000 seaweed && \
     adduser -D -u 1000 -G seaweed seaweed
 

docker/Dockerfile.rocksdb_large_local

@@ -17,7 +17,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
 COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
 
 # Install dependencies and create non-root user
-RUN apk add --no-cache fuse snappy gflags curl tmux su-exec && \
+RUN apk upgrade --no-cache zlib && \
+    apk add --no-cache fuse snappy gflags curl tmux su-exec && \
     addgroup -g 1000 seaweed && \
     adduser -D -u 1000 -G seaweed seaweed
 

go.mod

@@ -83,10 +83,10 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/viant/ptrie v1.0.1
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/scram v1.2.0
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	go.etcd.io/etcd/client/v3 v3.6.7
+	go.etcd.io/etcd/client/v3 v3.6.9
 	go.mongodb.org/mongo-driver v1.17.9
 	go.opencensus.io v0.24.0 // indirect
 	gocloud.dev v0.45.0
@@ -94,7 +94,7 @@ require (
 	gocloud.dev/pubsub/rabbitpubsub v0.45.0
 	golang.org/x/crypto v0.49.0
 	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
-	golang.org/x/image v0.36.0
+	golang.org/x/image v0.38.0
 	golang.org/x/net v0.51.0
 	golang.org/x/oauth2 v0.36.0
 	golang.org/x/sys v0.42.0
@@ -114,7 +114,7 @@ require (
 )
 
 require (
-	cloud.google.com/go/kms v1.25.0
+	cloud.google.com/go/kms v1.26.0
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
 	github.com/Jille/raft-grpc-transport v1.6.1
 	github.com/ThreeDotsLabs/watermill v1.5.1
@@ -124,13 +124,13 @@ require (
 	github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
 	github.com/arangodb/go-driver v1.6.9
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.41.4
-	github.com/aws/aws-sdk-go-v2/config v1.32.9
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.12
+	github.com/aws/aws-sdk-go-v2 v1.41.5
+	github.com/aws/aws-sdk-go-v2/config v1.32.13
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.13
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
-	github.com/getsentry/sentry-go v0.43.0
+	github.com/getsentry/sentry-go v0.44.1
 	github.com/go-ldap/ldap/v3 v3.4.13
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
@@ -157,7 +157,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
 	github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
-	go.etcd.io/etcd/client/pkg/v3 v3.6.7
+	go.etcd.io/etcd/client/pkg/v3 v3.6.9
 	go.uber.org/atomic v1.11.0
 	golang.org/x/sync v0.20.0
 	golang.org/x/tools/godoc v0.1.0-deprecated
@@ -181,7 +181,7 @@ require (
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
 	github.com/apache/thrift v0.22.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/bazelbuild/rules_go v0.46.0 // indirect
 	github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
@@ -324,21 +324,21 @@ require (
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.41.9
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
 	github.com/aws/smithy-go v1.24.2
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.3 // indirect
@@ -486,7 +486,7 @@ require (
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
 	go.etcd.io/bbolt v1.4.3 // indirect
-	go.etcd.io/etcd/api/v3 v3.6.7 // indirect
+	go.etcd.io/etcd/api/v3 v3.6.9 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect

go.sum

@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
 cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
 cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
 cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
-cloud.google.com/go/kms v1.25.0 h1:gVqvGGUmz0nYCmtoxWmdc1wli2L1apgP8U4fghPGSbQ=
-cloud.google.com/go/kms v1.25.0/go.mod h1:XIdHkzfj0bUO3E+LvwPg+oc7s58/Ns8Nd8Sdtljihbk=
+cloud.google.com/go/kms v1.26.0 h1:cK9mN2cf+9V63D3H1f6koxTatWy39aTI/hCjz1I+adU=
+cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58=
 cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
 cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
 cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
@@ -714,48 +714,48 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
-github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
+github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
+github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A=
-github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
+github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
+github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
 github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
 github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
 github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
 github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
@@ -1051,8 +1051,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
 github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
 github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
 github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
-github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
-github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
+github.com/getsentry/sentry-go v0.44.1 h1:/cPtrA5qB7uMRrhgSn9TYtcEF36auGP3Y6+ThvD/yaI=
+github.com/getsentry/sentry-go v0.44.1/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
 github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
@@ -1838,8 +1838,6 @@ github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1Ivohy
 github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
 github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564 h1:TgxPraf1NmF6XTcUG53ULpLQrKvhtUJxQ3hyekxSDNQ=
 github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
-github.com/seaweedfs/go-fuse/v2 v2.9.1 h1:gnKmfrKreCRGJmekGz5WMnNZqXEf9s9+V2hdWQdvx88=
-github.com/seaweedfs/go-fuse/v2 v2.9.1/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
 github.com/seaweedfs/go-fuse/v2 v2.9.2 h1:IfP/yFjLGO4rALcJY2Gb39PlebHxLnj7dkIiQAjFres=
 github.com/seaweedfs/go-fuse/v2 v2.9.2/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
 github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
@@ -2038,8 +2036,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
-github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
+github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -2103,12 +2101,12 @@ go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
 go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
 go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
-go.etcd.io/etcd/api/v3 v3.6.7 h1:7BNJ2gQmc3DNM+9cRkv7KkGQDayElg8x3X+tFDYS+E0=
-go.etcd.io/etcd/api/v3 v3.6.7/go.mod h1:xJ81TLj9hxrYYEDmXTeKURMeY3qEDN24hqe+q7KhbnI=
-go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/AhBVs=
-go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
-go.etcd.io/etcd/client/v3 v3.6.7 h1:9WqA5RpIBtdMxAy1ukXLAdtg2pAxNqW5NUoO2wQrE6U=
-go.etcd.io/etcd/client/v3 v3.6.7/go.mod h1:2XfROY56AXnUqGsvl+6k29wrwsSbEh1lAouQB1vHpeE=
+go.etcd.io/etcd/api/v3 v3.6.9 h1:UA7iKfEW1AzgihcBSGXci2kDGQiokSq41F9HMCI/RTI=
+go.etcd.io/etcd/api/v3 v3.6.9/go.mod h1:csEk/qTfxKL36NqJdU15Tgtl65A8dyEY2BYo7PRsIwk=
+go.etcd.io/etcd/client/pkg/v3 v3.6.9 h1:T8nuk8Lz64C+Hzb0coBFLMSlVSQZBpAtFk46swdM1DA=
+go.etcd.io/etcd/client/pkg/v3 v3.6.9/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
+go.etcd.io/etcd/client/v3 v3.6.9 h1:3X555hQXmhRr27O37wls53g68CpUiPOiHXrZfz2Al+o=
+go.etcd.io/etcd/client/v3 v3.6.9/go.mod h1:KO7H1HLYh1qaljuVZJQwBFk1lRce6pJzt+C81GEnrlM=
 go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
 go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -2242,8 +2240,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
 golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
-golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
-golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
+golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
+golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.17"
+appVersion: "4.18"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.17.0
+version: 4.18.0

other/java/client/src/main/proto/filer.proto

@@ -84,6 +84,8 @@ service SeaweedFiler {
     // distributed lock management internal use only
     rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
     }
+    rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
+    }
 }
 
 //////////////////////////////////////////////////
@@ -522,6 +524,7 @@ message LockResponse {
     string lock_owner = 2;
     string lock_host_moved_to = 3;
     string error = 4;
+    int64 generation = 5;
 }
 message UnlockRequest {
     string name = 1;
@@ -544,12 +547,26 @@ message Lock {
     string renew_token = 2;
     int64 expired_at_ns = 3;
     string owner = 4;
+    int64 generation = 5;
+    bool is_backup = 6;
+    int64 seq = 7;
 }
 message TransferLocksRequest {
     repeated Lock locks = 1;
 }
 message TransferLocksResponse {
 }
+message ReplicateLockRequest {
+    string name = 1;
+    string renew_token = 2;
+    int64 expired_at_ns = 3;
+    string owner = 4;
+    int64 generation = 5;
+    bool is_unlock = 6;
+    int64 seq = 7;
+}
+message ReplicateLockResponse {
+}
 
 //////////////////////////////////////////////////
 // StreamMutateEntry: ordered bidirectional streaming for all filer mutations.

test/fuse_integration/git_operations_test.go

@@ -442,6 +442,20 @@ func resetToCommitWithRecovery(t *testing.T, bareRepo, localClone, mountClone, c
 			}
 			continue
 		}
+		// The kernel dcache can drop the FUSE entry moments after a
+		// successful check.  Wait briefly and re-verify to confirm the
+		// directory has stabilised before returning to the caller.
+		time.Sleep(1 * time.Second)
+		refreshDirEntry(t, mountClone)
+		if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
+			lastErr = fmt.Errorf("post-reset stabilisation check failed: %w", err)
+			if attempt < maxAttempts {
+				t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
+				os.RemoveAll(mountClone)
+				time.Sleep(2 * time.Second)
+			}
+			continue
+		}
 		return
 	}
 	require.NoError(t, lastErr, "git reset --hard %s failed after %d recovery attempts", commit, maxAttempts)
@@ -469,6 +483,10 @@ func tryPullFromCommit(t *testing.T, bareRepo, localClone, cloneDir, fromCommit
 	}
 	refreshDirEntry(t, cloneDir)
 
+	// Let the dcache stabilise before proceeding.
+	time.Sleep(1 * time.Second)
+	refreshDirEntry(t, cloneDir)
+
 	head, err := tryGitCommand(cloneDir, "rev-parse", "HEAD")
 	if err != nil {
 		return fmt.Errorf("rev-parse after reset: %w", err)

test/kafka/go.mod

@@ -14,7 +14,7 @@ require (
 replace github.com/seaweedfs/seaweedfs => ../../
 
 require (
-	cloud.google.com/go/auth v0.17.0 // indirect
+	cloud.google.com/go/auth v0.18.1 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
@@ -45,19 +45,19 @@ require (
 	github.com/aws/aws-sdk-go v1.55.8 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.41.4 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.32.7 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.32.9 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
@@ -124,8 +124,8 @@ require (
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
-	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
+	github.com/googleapis/gax-go/v2 v2.17.0 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gorilla/schema v1.4.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -147,9 +147,9 @@ require (
 	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
 	github.com/karlseguin/ccache/v2 v2.0.8 // indirect
-	github.com/klauspost/compress v1.18.4 // indirect
+	github.com/klauspost/compress v1.18.5 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
-	github.com/klauspost/reedsolomon v1.13.0 // indirect
+	github.com/klauspost/reedsolomon v1.13.3 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
@@ -237,7 +237,7 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	github.com/zeebo/errs v1.4.0 // indirect
-	github.com/zeebo/xxh3 v1.0.2 // indirect
+	github.com/zeebo/xxh3 v1.1.0 // indirect
 	go.etcd.io/bbolt v1.4.3 // indirect
 	go.mongodb.org/mongo-driver v1.17.9 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
@@ -247,18 +247,18 @@ require (
 	go.opentelemetry.io/otel/trace v1.40.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.48.0 // indirect
-	golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
-	golang.org/x/image v0.36.0 // indirect
-	golang.org/x/net v0.49.0 // indirect
+	golang.org/x/crypto v0.49.0 // indirect
+	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
+	golang.org/x/image v0.38.0 // indirect
+	golang.org/x/net v0.51.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.42.0 // indirect
-	golang.org/x/term v0.40.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
+	golang.org/x/term v0.41.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
-	google.golang.org/api v0.258.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
+	google.golang.org/api v0.267.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
 	google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect

test/kafka/go.sum

@@ -13,8 +13,8 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
 cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4=
-cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ=
+cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
+cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
@@ -116,8 +116,8 @@ github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1h
 github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
-github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
-github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY=
+github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A=
+github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI=
 github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
 github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
@@ -130,18 +130,18 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDk
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZifkk6vEGl/1Qor0=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
 github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
@@ -385,12 +385,12 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
 github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ=
-github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
+github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
+github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
-github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
+github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
+github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
 github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
 github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
@@ -467,12 +467,12 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt
 github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
-github.com/klauspost/reedsolomon v1.13.0 h1:E0Cmgf2kMuhZTj6eefnvpKC4/Q4jhCi9YIjcZjK4arc=
-github.com/klauspost/reedsolomon v1.13.0/go.mod h1:ggJT9lc71Vu+cSOPBlxGvBN6TfAS77qB4fp8vJ05NSA=
+github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
+github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 h1:CjEMN21Xkr9+zwPmZPaJJw+apzVbjGL5uK/6g9Q2jGU=
 github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988/go.mod h1:/agobYum3uo/8V6yPVnq+R82pyVGCeuWW5arT4Txn8A=
 github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 h1:FHVoZMOVRA+6/y4yRlbiR3WvsrOcKBd/f64H7YiWR2U=
@@ -739,8 +739,8 @@ github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
 github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
 github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
 github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
-github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
-github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
+github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
+github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
 go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
 go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
@@ -787,8 +787,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
-golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
+golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -799,12 +799,12 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
-golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
+golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
+golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
-golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
+golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
+golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -869,8 +869,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
-golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
+golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
+golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -894,8 +894,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -959,8 +959,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
-golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
-golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
+golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
+golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -976,8 +976,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1030,14 +1030,14 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
-golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
+golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
+golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
-gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -1054,8 +1054,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.258.0 h1:IKo1j5FBlN74fe5isA2PVozN3Y5pwNKriEgAXPOkDAc=
-google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww=
+google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
+google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1091,12 +1091,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 h1:dDbsTLIK7EzwUq36kCSAsk0slouq/S0tWHeeGi97cD8=
-google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846/go.mod h1:PP0g88Dz3C7hRAfbQCQggeWAXjuqGsNPLE4s7jh0RGU=
-google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
-google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
+google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
+google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
+google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
+google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

test/plugin_workers/fake_volume_server.go

@@ -11,9 +11,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
 // VolumeServer provides a minimal volume server for erasure coding tests.
@@ -196,12 +198,25 @@ func (v *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream vo
 	defer file.Close()
 
 	buf := make([]byte, 64*1024)
+	remaining := int64(req.GetStopOffset())
 	for {
-		n, readErr := file.Read(buf)
+		if remaining == 0 {
+			break
+		}
+
+		readBuf := buf
+		if remaining > 0 && remaining < int64(len(buf)) {
+			readBuf = buf[:remaining]
+		}
+
+		n, readErr := file.Read(readBuf)
 		if n > 0 {
-			if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: buf[:n]}); err != nil {
+			if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: readBuf[:n]}); err != nil {
 				return err
 			}
+			if remaining > 0 {
+				remaining -= int64(n)
+			}
 		}
 		if readErr == io.EOF {
 			break
@@ -307,10 +322,21 @@ func (v *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_ser
 	v.mu.Lock()
 	v.readFileStatusCalls++
 	v.mu.Unlock()
+
+	datInfo, err := os.Stat(v.filePath(req.VolumeId, ".dat"))
+	if err != nil {
+		return nil, err
+	}
+
+	idxInfo, err := os.Stat(v.filePath(req.VolumeId, ".idx"))
+	if err != nil {
+		return nil, err
+	}
+
 	return &volume_server_pb.ReadVolumeFileStatusResponse{
 		VolumeId:    req.VolumeId,
-		DatFileSize: 1024,
-		IdxFileSize: 16,
+		DatFileSize: uint64(datInfo.Size()),
+		IdxFileSize: uint64(idxInfo.Size()),
 		FileCount:   1,
 	}, nil
 }
@@ -349,7 +375,27 @@ func (v *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, strea
 	v.volumeCopyCalls++
 	v.mu.Unlock()
 
-	if err := stream.Send(&volume_server_pb.VolumeCopyResponse{ProcessedBytes: 1024}); err != nil {
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	var statusResp *volume_server_pb.ReadVolumeFileStatusResponse
+	if err := operation.WithVolumeServerClient(false, pb.ServerAddress(req.SourceDataNode), dialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			var readErr error
+			statusResp, readErr = client.ReadVolumeFileStatus(stream.Context(), &volume_server_pb.ReadVolumeFileStatusRequest{
+				VolumeId: req.VolumeId,
+			})
+			return readErr
+		}); err != nil {
+		return err
+	}
+
+	if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".dat", statusResp.DatFileSize, dialOption); err != nil {
+		return err
+	}
+	if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".idx", statusResp.IdxFileSize, dialOption); err != nil {
+		return err
+	}
+
+	if err := stream.Send(&volume_server_pb.VolumeCopyResponse{ProcessedBytes: int64(statusResp.DatFileSize + statusResp.IdxFileSize)}); err != nil {
 		return err
 	}
 	return stream.Send(&volume_server_pb.VolumeCopyResponse{LastAppendAtNs: uint64(time.Now().UnixNano())})
@@ -368,3 +414,44 @@ func (v *VolumeServer) VolumeTailReceiver(ctx context.Context, req *volume_serve
 	v.mu.Unlock()
 	return &volume_server_pb.VolumeTailReceiverResponse{}, nil
 }
+
+func (v *VolumeServer) copyRemoteFile(ctx context.Context, sourceDataNode string, volumeID uint32, ext string, fileSize uint64, dialOption grpc.DialOption) error {
+	path := v.filePath(volumeID, ext)
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return err
+	}
+
+	file, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	return operation.WithVolumeServerClient(true, pb.ServerAddress(sourceDataNode), dialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			stream, err := client.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+				VolumeId:   volumeID,
+				Ext:        ext,
+				StopOffset: fileSize,
+			})
+			if err != nil {
+				return err
+			}
+
+			for {
+				resp, recvErr := stream.Recv()
+				if recvErr == io.EOF {
+					return nil
+				}
+				if recvErr != nil {
+					return recvErr
+				}
+				if len(resp.FileContent) == 0 {
+					continue
+				}
+				if _, err := file.Write(resp.FileContent); err != nil {
+					return err
+				}
+			}
+		})
+}

test/plugin_workers/volume_balance/execution_test.go

@@ -16,6 +16,8 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+const testVolumeDatSize = 1 * 1024 * 1024
+
 func TestVolumeBalanceExecutionIntegration(t *testing.T) {
 	volumeID := uint32(303)
 
@@ -31,6 +33,7 @@ func TestVolumeBalanceExecutionIntegration(t *testing.T) {
 
 	source := pluginworkers.NewVolumeServer(t, "")
 	target := pluginworkers.NewVolumeServer(t, "")
+	pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), volumeID, testVolumeDatSize)
 
 	job := &plugin_pb.JobSpec{
 		JobId:   fmt.Sprintf("balance-job-%d", volumeID),
@@ -84,6 +87,9 @@ func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
 
 	// Build a batch job with 3 volume moves from source → target.
 	volumeIDs := []uint32{401, 402, 403}
+	for _, vid := range volumeIDs {
+		pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), vid, testVolumeDatSize)
+	}
 	moves := make([]*worker_pb.BalanceMoveSpec, len(volumeIDs))
 	for i, vid := range volumeIDs {
 		moves[i] = &worker_pb.BalanceMoveSpec{
@@ -139,10 +145,11 @@ func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
 		require.True(t, deletedVols[vid], "volume %d should have been deleted from source", vid)
 	}
 
-	// Pre-delete verification should have called ReadVolumeFileStatus on both
-	// source and target for each volume.
-	require.Equal(t, len(volumeIDs), source.ReadFileStatusCount(),
-		"each move should read source volume status before delete")
+	// Each move reads source status once before copy and once inside the
+	// target's fake VolumeCopy implementation, then reads target status once
+	// before deleting the source.
+	require.Equal(t, len(volumeIDs)*2, source.ReadFileStatusCount(),
+		"each move should read source volume status before copy and during target copy")
 	require.Equal(t, len(volumeIDs), target.ReadFileStatusCount(),
 		"each move should read target volume status before delete")
 

test/s3/spark/setup_test.go

@@ -4,8 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"math/rand"
-	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -96,9 +94,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}
 
-	env.masterPort = mustFreePort(t, "Master")
-	env.filerPort = mustFreePort(t, "Filer")
-	env.s3Port = mustFreePort(t, "S3")
+	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
+	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
+	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
 
 	bindIP := testutil.FindBindIP()
 	iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)
@@ -135,14 +133,14 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	}
 	registerMiniProcess(env.masterProcess)
 
-	if !waitForPort(env.masterPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
 	}
-	if !waitForPort(env.filerPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
 	}
-	if !waitForPort(env.s3Port, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
 	}
 }
 
@@ -208,46 +206,6 @@ func (env *TestEnvironment) Cleanup(t *testing.T) {
 	}
 }
 
-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-
-	for i := 0; i < 200; i++ {
-		port := 20000 + rand.Intn(30000)
-		listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port))
-		if err != nil {
-			continue
-		}
-		_ = listener.Close()
-
-		grpcPort := port + 10000
-		if grpcPort > 65535 {
-			continue
-		}
-		grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", grpcPort))
-		if err != nil {
-			continue
-		}
-		_ = grpcListener.Close()
-		return port
-	}
-
-	t.Fatalf("failed to get free port for %s", name)
-	return 0
-}
-
-func waitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
-		if err == nil {
-			_ = conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}
-
 func runSparkPyScript(t *testing.T, container testcontainers.Container, script string, s3Port int) (int, string) {
 	t.Helper()
 

test/s3tables/catalog_risingwave/setup_test.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -115,11 +114,11 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}
 
-	env.masterPort = mustFreePort(t, "Master")
-	env.filerPort = mustFreePort(t, "Filer")
-	env.s3Port = mustFreePort(t, "S3")
-	env.icebergRestPort = mustFreePort(t, "Iceberg")
-	env.risingwavePort = mustFreePort(t, "RisingWave")
+	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
+	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
+	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
+	env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")
+	env.risingwavePort = testutil.MustFreeMiniPort(t, "RisingWave")
 
 	env.bindIP = testutil.FindBindIP()
 
@@ -161,62 +160,20 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	registerMiniProcess(env.masterProcess)
 
 	// Wait for all services to be ready
-	if !waitForPort(env.masterPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
 	}
-	if !waitForPort(env.filerPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
 	}
-	if !waitForPort(env.s3Port, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
 	}
-	if !waitForPort(env.icebergRestPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - iceberg rest port %d not listening", env.icebergRestPort)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/v1/config", env.icebergRestPort), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - iceberg rest endpoint http://127.0.0.1:%d/v1/config not responding", env.icebergRestPort)
 	}
 }
 
-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-	minPort := 10000
-	maxPort := 55000 // Ensure port+10000 < 65535
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-
-	for i := 0; i < 1000; i++ {
-		port := minPort + r.Intn(maxPort-minPort)
-
-		// Check http port
-		ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
-		if err != nil {
-			continue
-		}
-		ln.Close()
-
-		// Check grpc port (weed mini uses port+10000)
-		ln2, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port+10000))
-		if err != nil {
-			continue
-		}
-		ln2.Close()
-
-		return port
-	}
-	t.Fatalf("failed to find a free port < %d for %s after 1000 attempts", maxPort, name)
-	return 0
-}
-
-func waitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
-		if err == nil {
-			conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}
-
 func (env *TestEnvironment) StartRisingWave(t *testing.T) {
 	t.Helper()
 
@@ -253,7 +210,7 @@ func (env *TestEnvironment) StartRisingWave(t *testing.T) {
 	}
 
 	// Wait for RisingWave port to be open on host
-	if !waitForPort(env.risingwavePort, 120*time.Second) {
+	if !testutil.WaitForPort(env.risingwavePort, 120*time.Second) {
 		t.Fatalf("timed out waiting for RisingWave port %d to be open", env.risingwavePort)
 	}
 

test/s3tables/catalog_spark/setup_test.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
-	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -91,10 +90,10 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}
 
-	env.masterPort = mustFreePort(t, "Master")
-	env.filerPort = mustFreePort(t, "Filer")
-	env.s3Port = mustFreePort(t, "S3")
-	env.icebergRestPort = mustFreePort(t, "Iceberg")
+	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
+	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
+	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
+	env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")
 
 	bindIP := testutil.FindBindIP()
 
@@ -127,58 +126,20 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	registerMiniProcess(env.masterProcess)
 
 	// Wait for all services to be ready
-	if !waitForPort(env.masterPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
 	}
-	if !waitForPort(env.filerPort, 15*time.Second) {
+	if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
 		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
 	}
-	if !waitForPort(env.s3Port, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
 	}
-	if !waitForPort(env.icebergRestPort, 15*time.Second) {
-		t.Fatalf("weed mini failed to start - iceberg rest port %d not listening", env.icebergRestPort)
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/v1/config", env.icebergRestPort), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - iceberg rest endpoint http://127.0.0.1:%d/v1/config not responding", env.icebergRestPort)
 	}
 }
 
-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-
-	for i := 0; i < 200; i++ {
-		port := 20000 + rand.Intn(30000)
-		listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port))
-		if err != nil {
-			continue
-		}
-		listener.Close()
-		grpcPort := port + 10000
-		if grpcPort > 65535 {
-			continue
-		}
-		grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", grpcPort))
-		if err != nil {
-			continue
-		}
-		grpcListener.Close()
-		return port
-	}
-	t.Fatalf("failed to get free port for %s", name)
-	return 0
-}
-
-func waitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
-		if err == nil {
-			conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}
-
 func (env *TestEnvironment) writeSparkConfig(t *testing.T, catalogBucket string) string {
 	t.Helper()
 

test/s3tables/testutil/docker.go

@@ -2,6 +2,7 @@ package testutil
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"net/http"
 	"os/exec"
@@ -64,3 +65,17 @@ func WaitForService(url string, timeout time.Duration) bool {
 		}
 	}
 }
+
+func WaitForPort(port int, timeout time.Duration) bool {
+	deadline := time.Now().Add(timeout)
+	address := fmt.Sprintf("127.0.0.1:%d", port)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", address, 500*time.Millisecond)
+		if err == nil {
+			_ = conn.Close()
+			return true
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return false
+}

test/s3tables/testutil/weed_mini.go

@@ -2,11 +2,16 @@ package testutil
 
 import (
 	"fmt"
+	"math/rand"
 	"net"
 	"os"
 	"path/filepath"
+	"testing"
+	"time"
 )
 
+const SeaweedMiniStartupTimeout = 45 * time.Second
+
 func FindBindIP() string {
 	addrs, err := net.InterfaceAddrs()
 	if err != nil {
@@ -54,3 +59,35 @@ func WriteIAMConfig(dir, accessKey, secretKey string) (string, error) {
 	}
 	return iamConfigPath, nil
 }
+
+func MustFreeMiniPort(t *testing.T, name string) int {
+	t.Helper()
+
+	const (
+		minPort = 10000
+		maxPort = 55000
+	)
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	for i := 0; i < 1000; i++ {
+		port := minPort + r.Intn(maxPort-minPort)
+
+		listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+		if err != nil {
+			continue
+		}
+		_ = listener.Close()
+
+		grpcPort := port + 10000
+		grpcListener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
+		if err != nil {
+			continue
+		}
+		_ = grpcListener.Close()
+
+		return port
+	}
+
+	t.Fatalf("failed to get free weed mini port for %s", name)
+	return 0
+}

test/volume_server/grpc/ec_decode_ecj_test.go

@@ -0,0 +1,294 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// TestEcDecodePreservesDeletedNeedles verifies that needles deleted via
+// VolumeEcBlobDelete (recorded in .ecj) are correctly excluded from the
+// decoded volume produced by VolumeEcShardsToVolume.
+func TestEcDecodePreservesDeletedNeedles(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const (
+		volumeID = uint32(140)
+		keyA     = uint64(990020)
+		cookieA  = uint32(0xDA001122)
+		keyB     = uint64(990021)
+		cookieB  = uint32(0xDA003344)
+	)
+
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fidA := framework.NewFileID(volumeID, keyA, cookieA)
+	fidB := framework.NewFileID(volumeID, keyB, cookieB)
+	payloadA := []byte("needle-A-should-be-deleted-after-decode")
+	payloadB := []byte("needle-B-should-survive-decode")
+
+	// Upload two needles.
+	for _, tc := range []struct {
+		fid     string
+		payload []byte
+	}{
+		{fidA, payloadA},
+		{fidB, payloadB},
+	} {
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, tc.payload)
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s: expected 201, got %d", tc.fid, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// EC encode.
+	_, err := client.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate: %v", err)
+	}
+
+	// Mount all data shards so the EC volume is usable.
+	_, err = client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID, Collection: "",
+		ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount: %v", err)
+	}
+
+	// Delete needle A via EC path (writes to .ecj).
+	_, err = client.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId: volumeID, Collection: "",
+		FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete needle A: %v", err)
+	}
+
+	// Unmount the normal volume so decode writes fresh files.
+	_, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeUnmount: %v", err)
+	}
+
+	// Decode EC shards back to a normal volume.
+	_, err = client.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsToVolume: %v", err)
+	}
+
+	// Re-mount the decoded volume.
+	_, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeMount: %v", err)
+	}
+
+	// Needle A should be gone (deleted via .ecj before decode).
+	respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA)
+	bodyA := framework.ReadAllAndClose(t, respA)
+	if respA.StatusCode >= 500 {
+		t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
+	}
+	if respA.StatusCode != http.StatusNotFound {
+		t.Fatalf("needle A should be 404 after decode, got %d", respA.StatusCode)
+	}
+
+	// Needle B should still be readable.
+	respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB)
+	bodyB := framework.ReadAllAndClose(t, respB)
+	if respB.StatusCode != http.StatusOK {
+		t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
+	}
+	if string(bodyB) != string(payloadB) {
+		t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
+	}
+}
+
+// TestEcDecodeCollectsEcjFromPeer verifies that .ecj deletion entries from a
+// peer server that contributes no new data shards are still collected during
+// decode. This is the regression test for the fix in collectEcShards that
+// always copies .ecj from every shard location.
+//
+// Scenario:
+//   - Server 0 holds all 10 data shards (decode target).
+//   - Server 1 holds a copy of shard 0 (no new shards for server 0).
+//   - A needle is deleted ONLY on server 1 (server 0's .ecj is empty).
+//   - During decode on server 0, server 1's .ecj must be collected and applied.
+func TestEcDecodeCollectsEcjFromPeer(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMultiVolumeClusterAuto(t, matrix.P1(), 2)
+	conn0, client0 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, client1 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	const (
+		volumeID = uint32(141)
+		keyA     = uint64(990030)
+		cookieA  = uint32(0xDB001122)
+		keyB     = uint64(990031)
+		cookieB  = uint32(0xDB003344)
+	)
+
+	// Allocate and upload on server 0.
+	framework.AllocateVolume(t, client0, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fidA := framework.NewFileID(volumeID, keyA, cookieA)
+	fidB := framework.NewFileID(volumeID, keyB, cookieB)
+	payloadB := []byte("needle-B-should-survive-peer-ecj-decode")
+
+	resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA, []byte("needle-A-deleted-on-peer"))
+	_ = framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload A: expected 201, got %d", resp.StatusCode)
+	}
+	resp = framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB, payloadB)
+	_ = framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload B: expected 201, got %d", resp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// EC encode on server 0.
+	_, err := client0.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate on server 0: %v", err)
+	}
+
+	// Build the SourceDataNode address for server 0 (format: host:adminPort.grpcPort).
+	sourceDataNode := cluster.VolumeAdminAddress(0) + "." +
+		strings.Split(cluster.VolumeGRPCAddress(0), ":")[1]
+
+	// Copy shard 0 + ecx + ecj from server 0 → server 1.
+	_, err = client1.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: sourceDataNode,
+		ShardIds:       []uint32{0},
+		CopyEcxFile:    true,
+		CopyEcjFile:    true,
+		CopyVifFile:    true,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsCopy 0→1: %v", err)
+	}
+
+	// Mount shard 0 on server 1 so the EC volume can accept deletions.
+	_, err = client1.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID, Collection: "",
+		ShardIds: []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount on server 1: %v", err)
+	}
+
+	// Delete needle A on server 1 only (creates .ecj entry on server 1).
+	_, err = client1.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId: volumeID, Collection: "",
+		FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete needle A on server 1: %v", err)
+	}
+
+	// Mount all data shards on server 0 (the decode target).
+	_, err = client0.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID, Collection: "",
+		ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount on server 0: %v", err)
+	}
+
+	// Collect .ecj from server 1 → server 0 with NO new shard IDs.
+	// This is the critical path: server 1 has shard 0 which server 0 already
+	// has, so needToCopyShardsInfo would be empty. Before the fix in
+	// collectEcShards, this copy would be skipped entirely, losing server 1's
+	// deletion entries.
+	server1DataNode := cluster.VolumeAdminAddress(1) + "." +
+		strings.Split(cluster.VolumeGRPCAddress(1), ":")[1]
+
+	_, err = client0.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: server1DataNode,
+		ShardIds:       []uint32{}, // No new shards — just .ecj.
+		CopyEcxFile:    false,
+		CopyEcjFile:    true,
+		CopyVifFile:    false,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsCopy .ecj from server 1→0: %v", err)
+	}
+
+	// Unmount the normal volume before decode.
+	_, err = client0.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeUnmount on server 0: %v", err)
+	}
+
+	// Decode on server 0. RebuildEcxFile should see needle A's deletion from
+	// the .ecj that was collected from server 1.
+	_, err = client0.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsToVolume on server 0: %v", err)
+	}
+
+	// Re-mount the decoded normal volume.
+	_, err = client0.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeMount on server 0: %v", err)
+	}
+
+	// Needle A should be gone — its deletion was in server 1's .ecj.
+	respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA)
+	bodyA := framework.ReadAllAndClose(t, respA)
+	if respA.StatusCode >= 500 {
+		t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
+	}
+	if respA.StatusCode != http.StatusNotFound {
+		t.Fatalf("needle A should be 404 (ecj from peer), got %d", respA.StatusCode)
+	}
+
+	// Needle B should still be readable.
+	respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB)
+	bodyB := framework.ReadAllAndClose(t, respB)
+	if respB.StatusCode != http.StatusOK {
+		t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
+	}
+	if string(bodyB) != string(payloadB) {
+		t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
+	}
+}

weed/admin/dash/admin_server.go

@@ -943,7 +943,7 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 			leaderCount++
 		}
 
-		masterMap[master.Address] = masterInfo
+		masterMap[masterInfo.Address] = masterInfo
 	}
 
 	// Then, get additional master information from Raft cluster
@@ -955,11 +955,11 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 
 		// Process each raft server
 		for _, server := range resp.ClusterServers {
-			address := server.Address
-			httpAddress := pb.ServerAddress(address).ToHttpAddress()
+			// Raft stores gRPC addresses, convert to HTTP address
+			httpAddress := pb.GrpcAddressToServerAddress(server.Address)
 
 			// Update existing master info or create new one
-			if masterInfo, exists := masterMap[address]; exists {
+			if masterInfo, exists := masterMap[httpAddress]; exists {
 				// Update existing master with raft data
 				masterInfo.IsLeader = server.IsLeader
 				masterInfo.Suffrage = server.Suffrage
@@ -970,7 +970,7 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
 					IsLeader: server.IsLeader,
 					Suffrage: server.Suffrage,
 				}
-				masterMap[address] = masterInfo
+				masterMap[httpAddress] = masterInfo
 			}
 
 			if server.IsLeader {
@@ -1646,159 +1646,6 @@ func (as *AdminServer) GetConfigPersistence() *ConfigPersistence {
 	return as.configPersistence
 }
 
-// convertJSONToMaintenanceConfig converts JSON map to protobuf MaintenanceConfig
-func convertJSONToMaintenanceConfig(jsonConfig map[string]interface{}) (*maintenance.MaintenanceConfig, error) {
-	config := &maintenance.MaintenanceConfig{}
-
-	// Helper function to get int32 from interface{}
-	getInt32 := func(key string) (int32, error) {
-		if val, ok := jsonConfig[key]; ok {
-			switch v := val.(type) {
-			case int:
-				return int32(v), nil
-			case int32:
-				return v, nil
-			case int64:
-				return int32(v), nil
-			case float64:
-				return int32(v), nil
-			default:
-				return 0, fmt.Errorf("invalid type for %s: expected number, got %T", key, v)
-			}
-		}
-		return 0, nil
-	}
-
-	// Helper function to get bool from interface{}
-	getBool := func(key string) bool {
-		if val, ok := jsonConfig[key]; ok {
-			if b, ok := val.(bool); ok {
-				return b
-			}
-		}
-		return false
-	}
-
-	var err error
-
-	// Convert basic fields
-	config.Enabled = getBool("enabled")
-
-	if config.ScanIntervalSeconds, err = getInt32("scan_interval_seconds"); err != nil {
-		return nil, err
-	}
-	if config.WorkerTimeoutSeconds, err = getInt32("worker_timeout_seconds"); err != nil {
-		return nil, err
-	}
-	if config.TaskTimeoutSeconds, err = getInt32("task_timeout_seconds"); err != nil {
-		return nil, err
-	}
-	if config.RetryDelaySeconds, err = getInt32("retry_delay_seconds"); err != nil {
-		return nil, err
-	}
-	if config.MaxRetries, err = getInt32("max_retries"); err != nil {
-		return nil, err
-	}
-	if config.CleanupIntervalSeconds, err = getInt32("cleanup_interval_seconds"); err != nil {
-		return nil, err
-	}
-	if config.TaskRetentionSeconds, err = getInt32("task_retention_seconds"); err != nil {
-		return nil, err
-	}
-
-	// Convert policy if present
-	if policyData, ok := jsonConfig["policy"]; ok {
-		if policyMap, ok := policyData.(map[string]interface{}); ok {
-			policy := &maintenance.MaintenancePolicy{}
-
-			if globalMaxConcurrent, err := getInt32FromMap(policyMap, "global_max_concurrent"); err != nil {
-				return nil, err
-			} else {
-				policy.GlobalMaxConcurrent = globalMaxConcurrent
-			}
-
-			if defaultRepeatIntervalSeconds, err := getInt32FromMap(policyMap, "default_repeat_interval_seconds"); err != nil {
-				return nil, err
-			} else {
-				policy.DefaultRepeatIntervalSeconds = defaultRepeatIntervalSeconds
-			}
-
-			if defaultCheckIntervalSeconds, err := getInt32FromMap(policyMap, "default_check_interval_seconds"); err != nil {
-				return nil, err
-			} else {
-				policy.DefaultCheckIntervalSeconds = defaultCheckIntervalSeconds
-			}
-
-			// Convert task policies if present
-			if taskPoliciesData, ok := policyMap["task_policies"]; ok {
-				if taskPoliciesMap, ok := taskPoliciesData.(map[string]interface{}); ok {
-					policy.TaskPolicies = make(map[string]*maintenance.TaskPolicy)
-
-					for taskType, taskPolicyData := range taskPoliciesMap {
-						if taskPolicyMap, ok := taskPolicyData.(map[string]interface{}); ok {
-							taskPolicy := &maintenance.TaskPolicy{}
-
-							taskPolicy.Enabled = getBoolFromMap(taskPolicyMap, "enabled")
-
-							if maxConcurrent, err := getInt32FromMap(taskPolicyMap, "max_concurrent"); err != nil {
-								return nil, err
-							} else {
-								taskPolicy.MaxConcurrent = maxConcurrent
-							}
-
-							if repeatIntervalSeconds, err := getInt32FromMap(taskPolicyMap, "repeat_interval_seconds"); err != nil {
-								return nil, err
-							} else {
-								taskPolicy.RepeatIntervalSeconds = repeatIntervalSeconds
-							}
-
-							if checkIntervalSeconds, err := getInt32FromMap(taskPolicyMap, "check_interval_seconds"); err != nil {
-								return nil, err
-							} else {
-								taskPolicy.CheckIntervalSeconds = checkIntervalSeconds
-							}
-
-							policy.TaskPolicies[taskType] = taskPolicy
-						}
-					}
-				}
-			}
-
-			config.Policy = policy
-		}
-	}
-
-	return config, nil
-}
-
-// Helper functions for map conversion
-func getInt32FromMap(m map[string]interface{}, key string) (int32, error) {
-	if val, ok := m[key]; ok {
-		switch v := val.(type) {
-		case int:
-			return int32(v), nil
-		case int32:
-			return v, nil
-		case int64:
-			return int32(v), nil
-		case float64:
-			return int32(v), nil
-		default:
-			return 0, fmt.Errorf("invalid type for %s: expected number, got %T", key, v)
-		}
-	}
-	return 0, nil
-}
-
-func getBoolFromMap(m map[string]interface{}, key string) bool {
-	if val, ok := m[key]; ok {
-		if b, ok := val.(bool); ok {
-			return b
-		}
-	}
-	return false
-}
-
 type collectionStats struct {
 	PhysicalSize int64
 	LogicalSize  int64

weed/admin/dash/bucket_management.go

@@ -361,26 +361,6 @@ func normalizeQuotaUnit(unit string) (string, error) {
 	}
 }
 
-// Helper function to convert bytes to appropriate unit and size
-func convertBytesToQuota(bytes int64) (int64, string) {
-	if bytes == 0 {
-		return 0, "MB"
-	}
-
-	// Convert to TB if >= 1TB
-	if bytes >= 1024*1024*1024*1024 && bytes%(1024*1024*1024*1024) == 0 {
-		return bytes / (1024 * 1024 * 1024 * 1024), "TB"
-	}
-
-	// Convert to GB if >= 1GB
-	if bytes >= 1024*1024*1024 && bytes%(1024*1024*1024) == 0 {
-		return bytes / (1024 * 1024 * 1024), "GB"
-	}
-
-	// Convert to MB (default)
-	return bytes / (1024 * 1024), "MB"
-}
-
 // SetBucketQuota sets the quota for a bucket
 func (s *AdminServer) SetBucketQuota(bucketName string, quotaBytes int64, quotaEnabled bool) error {
 	return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

weed/admin/dash/ec_shard_management.go

@@ -506,18 +506,6 @@ func getShardCount(ecIndexBits uint32) int {
 	return count
 }
 
-// getMissingShards returns a slice of missing shard IDs for a volume
-// Assumes default 10+4 EC configuration (14 total shards)
-func getMissingShards(ecIndexBits uint32) []int {
-	var missing []int
-	for i := 0; i < erasure_coding.TotalShardsCount; i++ {
-		if (ecIndexBits & (1 << uint(i))) == 0 {
-			missing = append(missing, i)
-		}
-	}
-	return missing
-}
-
 // sortEcShards sorts EC shards based on the specified field and order
 func sortEcShards(shards []EcShardWithInfo, sortBy string, sortOrder string) {
 	sort.Slice(shards, func(i, j int) bool {

weed/admin/dash/mq_management.go

@@ -430,67 +430,6 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
 	return offsets, nil
 }
 
-// convertRecordTypeToSchemaFields converts a protobuf RecordType to SchemaFieldInfo slice
-func convertRecordTypeToSchemaFields(recordType *schema_pb.RecordType) []SchemaFieldInfo {
-	var schemaFields []SchemaFieldInfo
-
-	if recordType == nil || recordType.Fields == nil {
-		return schemaFields
-	}
-
-	for _, field := range recordType.Fields {
-		schemaField := SchemaFieldInfo{
-			Name:     field.Name,
-			Type:     getFieldTypeString(field.Type),
-			Required: field.IsRequired,
-		}
-		schemaFields = append(schemaFields, schemaField)
-	}
-
-	return schemaFields
-}
-
-// getFieldTypeString converts a protobuf Type to a human-readable string
-func getFieldTypeString(fieldType *schema_pb.Type) string {
-	if fieldType == nil {
-		return "unknown"
-	}
-
-	switch kind := fieldType.Kind.(type) {
-	case *schema_pb.Type_ScalarType:
-		return getScalarTypeString(kind.ScalarType)
-	case *schema_pb.Type_RecordType:
-		return "record"
-	case *schema_pb.Type_ListType:
-		elementType := getFieldTypeString(kind.ListType.ElementType)
-		return fmt.Sprintf("list<%s>", elementType)
-	default:
-		return "unknown"
-	}
-}
-
-// getScalarTypeString converts a protobuf ScalarType to a string
-func getScalarTypeString(scalarType schema_pb.ScalarType) string {
-	switch scalarType {
-	case schema_pb.ScalarType_BOOL:
-		return "bool"
-	case schema_pb.ScalarType_INT32:
-		return "int32"
-	case schema_pb.ScalarType_INT64:
-		return "int64"
-	case schema_pb.ScalarType_FLOAT:
-		return "float"
-	case schema_pb.ScalarType_DOUBLE:
-		return "double"
-	case schema_pb.ScalarType_BYTES:
-		return "bytes"
-	case schema_pb.ScalarType_STRING:
-		return "string"
-	default:
-		return "unknown"
-	}
-}
-
 // convertTopicPublishers converts protobuf TopicPublisher slice to PublisherInfo slice
 func convertTopicPublishers(publishers []*mq_pb.TopicPublisher) []PublisherInfo {
 	publisherInfos := make([]PublisherInfo, 0, len(publishers))

weed/admin/dash/plugin_api.go

@@ -2,8 +2,6 @@ package dash
 
 import (
 	"context"
-	"crypto/rand"
-	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -851,43 +849,6 @@ func normalizeTimeout(timeoutSeconds int, defaultTimeout, maxTimeout time.Durati
 	return timeout
 }
 
-func buildJobSpecFromProposal(jobType string, proposal *plugin_pb.JobProposal, index int) *plugin_pb.JobSpec {
-	now := timestamppb.Now()
-	suffix := make([]byte, 4)
-	if _, err := rand.Read(suffix); err != nil {
-		// Fallback to simpler ID if rand fails
-		suffix = []byte(fmt.Sprintf("%d", index))
-	}
-	jobID := fmt.Sprintf("%s-%d-%s", jobType, now.AsTime().UnixNano(), hex.EncodeToString(suffix))
-
-	jobSpec := &plugin_pb.JobSpec{
-		JobId:      jobID,
-		JobType:    jobType,
-		Priority:   plugin_pb.JobPriority_JOB_PRIORITY_NORMAL,
-		CreatedAt:  now,
-		Labels:     make(map[string]string),
-		Parameters: make(map[string]*plugin_pb.ConfigValue),
-		DedupeKey:  "",
-	}
-
-	if proposal != nil {
-		jobSpec.Summary = proposal.Summary
-		jobSpec.Detail = proposal.Detail
-		if proposal.Priority != plugin_pb.JobPriority_JOB_PRIORITY_UNSPECIFIED {
-			jobSpec.Priority = proposal.Priority
-		}
-		jobSpec.DedupeKey = proposal.DedupeKey
-		jobSpec.Parameters = plugin.CloneConfigValueMap(proposal.Parameters)
-		if proposal.Labels != nil {
-			for k, v := range proposal.Labels {
-				jobSpec.Labels[k] = v
-			}
-		}
-	}
-
-	return jobSpec
-}
-
 func applyDescriptorDefaultsToPersistedConfig(
 	config *plugin_pb.PersistedJobTypeConfig,
 	descriptor *plugin_pb.JobTypeDescriptor,

weed/admin/dash/plugin_api_test.go

@@ -115,32 +115,6 @@ func TestExpirePluginJobAPI(t *testing.T) {
 	})
 }
 
-func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
-	t.Parallel()
-
-	proposal := &plugin_pb.JobProposal{
-		ProposalId: "vacuum-2",
-		DedupeKey:  "vacuum:2",
-		JobType:    "vacuum",
-	}
-
-	jobA := buildJobSpecFromProposal("vacuum", proposal, 0)
-	jobB := buildJobSpecFromProposal("vacuum", proposal, 1)
-
-	if jobA.JobId == proposal.ProposalId {
-		t.Fatalf("job id must not reuse proposal id: %s", jobA.JobId)
-	}
-	if jobB.JobId == proposal.ProposalId {
-		t.Fatalf("job id must not reuse proposal id: %s", jobB.JobId)
-	}
-	if jobA.JobId == jobB.JobId {
-		t.Fatalf("job ids must be unique across jobs: %s", jobA.JobId)
-	}
-	if jobA.DedupeKey != proposal.DedupeKey {
-		t.Fatalf("dedupe key must be preserved: got=%s want=%s", jobA.DedupeKey, proposal.DedupeKey)
-	}
-}
-
 func TestApplyDescriptorDefaultsToPersistedConfigBackfillsAdminDefaults(t *testing.T) {
 	t.Parallel()
 

weed/admin/dash/worker_grpc_server.go

@@ -108,6 +108,8 @@ func (s *WorkerGrpcServer) StartWithTLS(port int) error {
 	go s.cleanupRoutine()
 	go s.activeLogFetchLoop()
 
+	pb.ServeGrpcOnLocalSocket(grpcServer, port)
+
 	// Start serving in a goroutine
 	go func() {
 		if err := s.grpcServer.Serve(listener); err != nil {
@@ -785,15 +787,6 @@ func (s *WorkerGrpcServer) RequestTaskLogsFromAllWorkers(taskID string, maxEntri
 	return results, nil
 }
 
-// convertTaskParameters converts task parameters to protobuf format
-func convertTaskParameters(params map[string]interface{}) map[string]string {
-	result := make(map[string]string)
-	for key, value := range params {
-		result[key] = fmt.Sprintf("%v", value)
-	}
-	return result
-}
-
 func findClientAddress(ctx context.Context) string {
 	// fmt.Printf("FromContext %+v\n", ctx)
 	pr, ok := peer.FromContext(ctx)

weed/cluster/lock_client.go

@@ -43,7 +43,8 @@ type LiveLock struct {
 	lc                  *LockClient
 	owner               string
 	lockTTL             time.Duration
-	consecutiveFailures int // Track connection failures to trigger fallback
+	consecutiveFailures int   // Track connection failures to trigger fallback
+	generation          int64 // fencing token from the lock server
 }
 
 // NewShortLivedLock creates a lock with a 5-second duration
@@ -214,6 +215,9 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
 		glog.V(4).Infof("LOCK: DistributedLock response - key=%s err=%v", lock.key, err)
 		if err == nil && resp != nil {
 			lock.renewToken = resp.RenewToken
+			if resp.Generation > 0 {
+				atomic.StoreInt64(&lock.generation, resp.Generation)
+			}
 			lock.consecutiveFailures = 0 // Reset failure counter on success
 			glog.V(4).Infof("LOCK: Got renewToken for key=%s", lock.key)
 		} else {
@@ -264,6 +268,12 @@ func (lock *LiveLock) LockOwner() string {
 	return lock.owner
 }
 
+// Generation returns the fencing token for this lock.
+// It increments on each fresh acquisition and stays the same on renewal.
+func (lock *LiveLock) Generation() int64 {
+	return atomic.LoadInt64(&lock.generation)
+}
+
 // IsLocked returns true if this instance currently holds the lock
 func (lock *LiveLock) IsLocked() bool {
 	return atomic.LoadInt32(&lock.isLocked) == 1

weed/cluster/lock_manager/distributed_lock_manager.go

@@ -13,10 +13,17 @@ const LiveLockTTL = time.Second * 7
 
 var NoLockServerError = fmt.Errorf("no lock server found")
 
+// ReplicateFunc is called to replicate a lock operation to a backup server.
+// The caller (filer server) provides this to avoid a circular dependency.
+// seq is a per-lock monotonic sequence number for causal ordering — the backup
+// rejects mutations with seq <= its current seq for that key.
+type ReplicateFunc func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool)
+
 type DistributedLockManager struct {
-	lockManager *LockManager
-	LockRing    *LockRing
-	Host        pb.ServerAddress
+	lockManager   *LockManager
+	LockRing      *LockRing
+	Host          pb.ServerAddress
+	ReplicateFn   ReplicateFunc // set by filer server after creation
 }
 
 func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
@@ -27,35 +34,51 @@ func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
 	}
 }
 
-func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (lockOwner string, renewToken string, movedTo pb.ServerAddress, err error) {
-	movedTo, err = dlm.findLockOwningFiler(key)
-	if err != nil {
-		return
-	}
-	if movedTo != dlm.Host {
-		return
-	}
-	lockOwner, renewToken, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
-	return
-}
-
-func (dlm *DistributedLockManager) findLockOwningFiler(key string) (movedTo pb.ServerAddress, err error) {
-	servers := dlm.LockRing.GetSnapshot()
-	if servers == nil {
+func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (lockOwner string, renewToken string, generation int64, movedTo pb.ServerAddress, err error) {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
 		err = NoLockServerError
 		return
 	}
-
-	movedTo = hashKeyToServer(key, servers)
+	if primary != dlm.Host {
+		// If this is a renewal (non-empty token) and we still hold the lock locally,
+		// serve it here rather than redirecting. This handles the window between
+		// ring update and lock transfer completion — the old primary remains
+		// authoritative for locks it still holds.
+		if token != "" {
+			if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
+				var seq int64
+				lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
+				if err == nil && renewToken != "" {
+					dlm.replicateToBackup(key, expiredAtNs, renewToken, owner, generation, seq, false)
+				}
+				return
+			}
+		}
+		movedTo = primary
+		return
+	}
+	var seq int64
+	lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
+	if err == nil && renewToken != "" {
+		dlm.replicateToBackup(key, expiredAtNs, renewToken, owner, generation, seq, false)
+	}
 	return
 }
 
 func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, movedTo pb.ServerAddress, err error) {
-	movedTo, err = dlm.findLockOwningFiler(key)
-	if err != nil {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
+		err = NoLockServerError
 		return
 	}
-	if movedTo != dlm.Host {
+	if primary != dlm.Host {
+		// If we still hold this lock locally, serve it here
+		if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup {
+			owner = lock.Owner
+			return
+		}
+		movedTo = primary
 		servers := dlm.LockRing.GetSnapshot()
 		glog.V(0).Infof("lock %s not on current %s but on %s from %v", key, dlm.Host, movedTo, servers)
 		return
@@ -65,26 +88,60 @@ func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, move
 }
 
 func (dlm *DistributedLockManager) Unlock(key string, token string) (movedTo pb.ServerAddress, err error) {
-	servers := dlm.LockRing.GetSnapshot()
-	if servers == nil {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
 		err = NoLockServerError
 		return
 	}
-
-	server := hashKeyToServer(key, servers)
-	if server != dlm.Host {
-		movedTo = server
+	if primary != dlm.Host {
+		// If we still hold this lock locally, serve the unlock here
+		if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
+			var isUnlocked bool
+			var generation int64
+			var seq int64
+			isUnlocked, generation, seq, err = dlm.lockManager.Unlock(key, token)
+			if isUnlocked {
+				dlm.replicateToBackup(key, 0, "", "", generation, seq, true)
+			}
+			return
+		}
+		movedTo = primary
 		return
 	}
-	_, err = dlm.lockManager.Unlock(key, token)
+	var isUnlocked bool
+	var generation int64
+	var seq int64
+	isUnlocked, generation, seq, err = dlm.lockManager.Unlock(key, token)
+	if isUnlocked {
+		dlm.replicateToBackup(key, 0, "", "", generation, seq, true)
+	}
 	return
 }
 
-// InsertLock is used to insert a lock to a server unconditionally
-// It is used when a server is down and the lock is moved to another server
-func (dlm *DistributedLockManager) InsertLock(key string, expiredAtNs int64, token string, owner string) {
-	dlm.lockManager.InsertLock(key, expiredAtNs, token, owner)
+// InsertLock is used to insert a lock to a server unconditionally.
+// It is used when a server is down and the lock is moved to another server.
+// After inserting, it replicates to the backup for this key.
+func (dlm *DistributedLockManager) InsertLock(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	if dlm.lockManager.InsertLock(key, expiredAtNs, token, owner, generation, seq) {
+		dlm.replicateToBackup(key, expiredAtNs, token, owner, generation, seq, false)
+	}
 }
+
+// InsertBackupLock inserts a lock as a backup copy, rejecting stale seq
+func (dlm *DistributedLockManager) InsertBackupLock(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	dlm.lockManager.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+}
+
+// RemoveBackupLock removes a backup lock unconditionally
+func (dlm *DistributedLockManager) RemoveBackupLock(key string) {
+	dlm.lockManager.RemoveLock(key)
+}
+
+// RemoveBackupLockIfSeq removes a local copy only if the incoming mutation is not older.
+func (dlm *DistributedLockManager) RemoveBackupLockIfSeq(key string, generation int64, seq int64) {
+	dlm.lockManager.RemoveBackupLockIfSeq(key, generation, seq)
+}
+
 func (dlm *DistributedLockManager) SelectNotOwnedLocks(servers []pb.ServerAddress) (locks []*Lock) {
 	return dlm.lockManager.SelectLocks(func(key string) bool {
 		server := hashKeyToServer(key, servers)
@@ -96,9 +153,40 @@ func (dlm *DistributedLockManager) CalculateTargetServer(key string, servers []p
 }
 
 func (dlm *DistributedLockManager) IsLocal(key string) bool {
-	servers := dlm.LockRing.GetSnapshot()
-	if len(servers) <= 1 {
+	primary := dlm.LockRing.GetPrimary(key)
+	if primary == "" {
 		return true
 	}
-	return hashKeyToServer(key, servers) == dlm.Host
+	return primary == dlm.Host
+}
+
+// AllLocks returns all non-expired locks on this node
+func (dlm *DistributedLockManager) AllLocks() []*Lock {
+	return dlm.lockManager.AllLocks()
+}
+
+// PromoteLock promotes a backup lock to primary
+func (dlm *DistributedLockManager) PromoteLock(key string) bool {
+	return dlm.lockManager.PromoteLock(key)
+}
+
+// DemoteLock demotes a primary lock to backup
+func (dlm *DistributedLockManager) DemoteLock(key string) bool {
+	return dlm.lockManager.DemoteLock(key)
+}
+
+// GetLock returns a copy of a lock if it exists
+func (dlm *DistributedLockManager) GetLock(key string) (*Lock, bool) {
+	return dlm.lockManager.GetLock(key)
+}
+
+// replicateToBackup asynchronously replicates a lock operation to the backup server
+func (dlm *DistributedLockManager) replicateToBackup(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+	_, backup := dlm.LockRing.GetPrimaryAndBackup(key)
+	if backup == "" {
+		return // single-server deployment, no backup
+	}
+	if dlm.ReplicateFn != nil {
+		go dlm.ReplicateFn(backup, key, expiredAtNs, token, owner, generation, seq, isUnlock)
+	}
 }

weed/cluster/lock_manager/distributed_lock_manager_test.go

@@ -0,0 +1,710 @@
+package lock_manager
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testCluster simulates a cluster of filer nodes with DLMs.
+// It wires up ReplicateFn so that replication calls arrive at the
+// correct peer's DLM, enabling end-to-end backup testing without gRPC.
+type testCluster struct {
+	mu    sync.Mutex
+	nodes map[pb.ServerAddress]*DistributedLockManager
+}
+
+func newTestCluster(hosts ...pb.ServerAddress) *testCluster {
+	c := &testCluster{nodes: make(map[pb.ServerAddress]*DistributedLockManager)}
+	servers := make([]pb.ServerAddress, len(hosts))
+	copy(servers, hosts)
+
+	for _, host := range hosts {
+		dlm := NewDistributedLockManager(host)
+		dlm.LockRing.SetSnapshot(servers, 0)
+		c.nodes[host] = dlm
+	}
+
+	// Wire up replication: each node's ReplicateFn calls the backup's DLM directly
+	for _, dlm := range c.nodes {
+		d := dlm // capture
+		d.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+			c.mu.Lock()
+			target, ok := c.nodes[server]
+			c.mu.Unlock()
+			if !ok {
+				return // server is down
+			}
+			if isUnlock {
+				target.RemoveBackupLockIfSeq(key, generation, seq)
+			} else {
+				target.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+			}
+		}
+	}
+
+	return c
+}
+
+func (c *testCluster) removeNode(host pb.ServerAddress) {
+	c.mu.Lock()
+	delete(c.nodes, host)
+	c.mu.Unlock()
+
+	// Update all remaining nodes' rings
+	remaining := c.getServers()
+	for _, dlm := range c.getNodes() {
+		dlm.LockRing.SetSnapshot(remaining, 0)
+	}
+}
+
+func (c *testCluster) addNode(host pb.ServerAddress) {
+	c.mu.Lock()
+	dlm := NewDistributedLockManager(host)
+	c.nodes[host] = dlm
+	c.mu.Unlock()
+
+	// Wire up replication
+	dlm.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+		c.mu.Lock()
+		target, ok := c.nodes[server]
+		c.mu.Unlock()
+		if !ok {
+			return
+		}
+		if isUnlock {
+			target.RemoveBackupLockIfSeq(key, generation, seq)
+		} else {
+			target.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+		}
+	}
+
+	servers := c.getServers()
+	for _, n := range c.getNodes() {
+		n.LockRing.SetSnapshot(servers, 0)
+	}
+}
+
+func (c *testCluster) getNodes() map[pb.ServerAddress]*DistributedLockManager {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	cp := make(map[pb.ServerAddress]*DistributedLockManager, len(c.nodes))
+	for k, v := range c.nodes {
+		cp[k] = v
+	}
+	return cp
+}
+
+func (c *testCluster) getServers() []pb.ServerAddress {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	var servers []pb.ServerAddress
+	for s := range c.nodes {
+		servers = append(servers, s)
+	}
+	return servers
+}
+
+func (c *testCluster) get(host pb.ServerAddress) *DistributedLockManager {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.nodes[host]
+}
+
+// acquireLock tries to acquire a lock on the correct primary node.
+// It follows redirects (movedTo) like a real client would.
+func (c *testCluster) acquireLock(key, owner string, ttl time.Duration) (renewToken string, generation int64, primaryHost pb.ServerAddress, err error) {
+	// Try any node first (simulates client connecting to seed filer)
+	for _, dlm := range c.getNodes() {
+		expiry := time.Now().Add(ttl).UnixNano()
+		var movedTo pb.ServerAddress
+		var lockErr error
+		_, renewToken, generation, movedTo, lockErr = dlm.LockWithTimeout(key, expiry, "", owner)
+		if movedTo != "" && movedTo != dlm.Host {
+			// Follow redirect
+			target := c.get(movedTo)
+			if target == nil {
+				err = fmt.Errorf("primary %s is down", movedTo)
+				return
+			}
+			_, renewToken, generation, _, lockErr = target.LockWithTimeout(key, expiry, "", owner)
+			if lockErr != nil {
+				err = lockErr
+				return
+			}
+			primaryHost = movedTo
+			// Wait briefly for async replication to complete
+			time.Sleep(10 * time.Millisecond)
+			return
+		}
+		if lockErr != nil {
+			err = lockErr
+			return
+		}
+		primaryHost = dlm.Host
+		time.Sleep(10 * time.Millisecond)
+		return
+	}
+	err = fmt.Errorf("no nodes available")
+	return
+}
+
+// renewLock renews a lock on the primary node
+func (c *testCluster) renewLock(key, owner, token string, ttl time.Duration, primaryHost pb.ServerAddress) (newToken string, generation int64, err error) {
+	target := c.get(primaryHost)
+	if target == nil {
+		err = fmt.Errorf("primary %s is down", primaryHost)
+		return
+	}
+	expiry := time.Now().Add(ttl).UnixNano()
+	var movedTo pb.ServerAddress
+	var lockErr error
+	_, newToken, generation, movedTo, lockErr = target.LockWithTimeout(key, expiry, token, owner)
+	if movedTo != "" && movedTo != primaryHost {
+		target = c.get(movedTo)
+		if target == nil {
+			err = fmt.Errorf("new primary %s is down", movedTo)
+			return
+		}
+		// Pass the existing token so the redirected renewal can match
+		// if the lock was already transferred to the new primary.
+		_, newToken, generation, _, lockErr = target.LockWithTimeout(key, expiry, token, owner)
+	}
+	err = lockErr
+	time.Sleep(10 * time.Millisecond)
+	return
+}
+
+// --- Test Cases ---
+
+func TestDLM_PrimaryCrash_BackupPromotes(t *testing.T) {
+	// Scenario: Lock is acquired, primary crashes, backup should have the lock
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-primary-crash"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+	require.NotEmpty(t, renewToken)
+
+	// Find the backup for this key
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+	require.NotEmpty(t, backup, "should have a backup server")
+
+	// Verify backup has the lock
+	backupDlm := cluster.get(backup)
+	backupLock, found := backupDlm.GetLock(key)
+	require.True(t, found, "backup should have the lock")
+	assert.True(t, backupLock.IsBackup, "lock on backup should be marked as backup")
+	assert.Equal(t, renewToken, backupLock.Token, "backup should have the same token")
+
+	// Crash the primary
+	cluster.removeNode(primaryHost)
+
+	// Simulate topology change: promote backup locks
+	for _, dlm := range cluster.getNodes() {
+		locks := dlm.AllLocks()
+		for _, lock := range locks {
+			newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if newPrimary == dlm.Host && lock.IsBackup {
+				dlm.PromoteLock(lock.Key)
+			}
+		}
+	}
+
+	// The backup should now be the primary
+	newPrimary := backupDlm.LockRing.GetPrimary(key)
+	assert.Equal(t, backup, newPrimary, "backup should be the new primary")
+
+	// The promoted lock should work — verify it's no longer a backup
+	promotedLock, found := backupDlm.GetLock(key)
+	require.True(t, found, "lock should still exist after promotion")
+	assert.False(t, promotedLock.IsBackup, "lock should be promoted to primary")
+
+	// Client should be able to renew with the same token on the new primary
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, backup)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "renewal on new primary should succeed")
+}
+
+func TestDLM_BackupCrash_PrimaryContinues(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-backup-crash"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Crash the backup
+	cluster.removeNode(backup)
+
+	// Primary should still work — renew the lock
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "primary should continue working after backup crash")
+
+	// Verify primary is still the primary for this key
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	assert.Equal(t, primaryHost, newPrimary)
+}
+
+func TestDLM_BothPrimaryAndBackupCrash(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-both-crash"
+	_, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Crash both
+	cluster.removeNode(primaryHost)
+	cluster.removeNode(backup)
+
+	// The lock is lost — the surviving node should be able to acquire it fresh
+	newToken, _, _, err := cluster.acquireLock(key, "owner2", 30*time.Second)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "new owner should acquire lock after both crash")
+}
+
+func TestDLM_RollingRestart(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire multiple locks
+	type lockState struct {
+		key, owner, token string
+		generation        int64
+		primary           pb.ServerAddress
+	}
+	locks := make([]lockState, 5)
+	for i := range locks {
+		key := fmt.Sprintf("rolling-lock-%d", i)
+		token, gen, primary, err := cluster.acquireLock(key, fmt.Sprintf("owner-%d", i), 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockState{key: key, owner: fmt.Sprintf("owner-%d", i), token: token, generation: gen, primary: primary}
+	}
+
+	// Rolling restart: remove and re-add each node one at a time.
+	// After removing a node, promote backups and re-replicate to new backups
+	// to maintain the invariant that each lock has a backup copy.
+	for _, host := range hosts {
+		cluster.removeNode(host)
+
+		// Simulate full OnDlmChangeSnapshot: promote backups and re-replicate
+		for _, dlm := range cluster.getNodes() {
+			for _, lock := range dlm.AllLocks() {
+				newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+				if newPrimary == dlm.Host && lock.IsBackup {
+					dlm.PromoteLock(lock.Key)
+				}
+			}
+			// Re-replicate all primary locks to their new backups
+			for _, lock := range dlm.AllLocks() {
+				newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+				if newPrimary == dlm.Host && !lock.IsBackup {
+					dlm.replicateToBackup(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq, false)
+				}
+			}
+		}
+
+		time.Sleep(10 * time.Millisecond)
+
+		// Re-add the node
+		cluster.addNode(host)
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	// After rolling restart, locks should survive via backup promotion
+	survivedCount := 0
+	for _, ls := range locks {
+		for _, dlm := range cluster.getNodes() {
+			lock, found := dlm.GetLock(ls.key)
+			if found && !lock.IsBackup {
+				survivedCount++
+				break
+			}
+		}
+	}
+	t.Logf("Locks survived rolling restart: %d / %d", survivedCount, len(locks))
+	require.Greater(t, survivedCount, 0, "at least some locks should survive a rolling restart via backup promotion")
+}
+
+func TestDLM_GenerationIncrementsOnNewAcquisition(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "gen-test-lock"
+
+	// Acquire lock — generation should be > 0
+	token1, gen1, primary, err := cluster.acquireLock(key, "owner1", 2*time.Second)
+	require.NoError(t, err)
+	assert.Greater(t, gen1, int64(0))
+
+	// Renew — generation should stay the same
+	token2, gen2, err := cluster.renewLock(key, "owner1", token1, 2*time.Second, primary)
+	require.NoError(t, err)
+	assert.Equal(t, gen1, gen2, "generation should not change on renewal")
+
+	// Let lock expire
+	time.Sleep(3 * time.Second)
+
+	// Re-acquire — generation should increment
+	_, gen3, _, err := cluster.acquireLock(key, "owner2", 30*time.Second)
+	require.NoError(t, err)
+	assert.Greater(t, gen3, gen1, "generation should increment on new acquisition")
+	_ = token2
+}
+
+func TestDLM_ReplicationFailure_PrimaryStillWorks(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Break replication by setting a no-op ReplicateFn on all nodes
+	for _, dlm := range cluster.getNodes() {
+		dlm.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+			// Simulate replication failure: do nothing
+		}
+	}
+
+	key := "repl-fail-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	// Primary should have the lock
+	primaryDlm := cluster.get(primaryHost)
+	lock, found := primaryDlm.GetLock(key)
+	require.True(t, found, "primary should have the lock")
+	assert.False(t, lock.IsBackup)
+
+	// Backup should NOT have it (replication failed)
+	_, backup := primaryDlm.LockRing.GetPrimaryAndBackup(key)
+	backupDlm := cluster.get(backup)
+	_, found = backupDlm.GetLock(key)
+	assert.False(t, found, "backup should not have the lock when replication fails")
+
+	// Primary should still be able to renew
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken)
+}
+
+func TestDLM_UnlockReplicatesToBackup(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "unlock-repl-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Verify backup has the lock
+	_, found := cluster.get(backup).GetLock(key)
+	require.True(t, found, "backup should have the lock")
+
+	// Unlock on primary
+	primaryDlm := cluster.get(primaryHost)
+	movedTo, err := primaryDlm.Unlock(key, renewToken)
+	require.NoError(t, err)
+	assert.Empty(t, movedTo)
+
+	// Wait for async replication
+	time.Sleep(20 * time.Millisecond)
+
+	// Backup should also have removed the lock
+	_, found = cluster.get(backup).GetLock(key)
+	assert.False(t, found, "backup should remove lock after unlock replication")
+}
+
+func TestDLM_TopologyChange_LockSurvivesServerAddition(t *testing.T) {
+	// Start with 2 servers, acquire lock, add a 3rd server
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "topo-add-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	// Add a new server
+	cluster.addNode("filer3:8888")
+	time.Sleep(20 * time.Millisecond)
+
+	// The lock should still be accessible — either the same primary or on a new one
+	// Try to renew on the original primary first
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	if newPrimary == primaryHost {
+		// Still on same primary
+		newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+		require.NoError(t, err)
+		assert.NotEmpty(t, newToken)
+	}
+	// If primary changed, the lock may need transfer — that's handled by OnDlmChangeSnapshot
+	// which is tested at the server level
+}
+
+func TestDLM_ConsistentHashing_MinimalDisruption(t *testing.T) {
+	// Verify that removing a server only affects locks on that server
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire 50 locks
+	type lockInfo struct {
+		key, token string
+		primary    pb.ServerAddress
+	}
+	locks := make([]lockInfo, 50)
+	for i := range locks {
+		key := fmt.Sprintf("min-disrupt-%d", i)
+		token, _, primary, err := cluster.acquireLock(key, "owner", 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockInfo{key: key, token: token, primary: primary}
+	}
+
+	// Count locks per server before removal
+	countBefore := make(map[pb.ServerAddress]int)
+	for _, l := range locks {
+		countBefore[l.primary]++
+	}
+	t.Logf("Lock distribution before: %v", countBefore)
+
+	// Remove filer2
+	cluster.removeNode("filer2:8888")
+
+	// Count how many locks changed primary
+	changed := 0
+	for _, l := range locks {
+		// Check where the lock should be now
+		for _, dlm := range cluster.getNodes() {
+			newPrimary := dlm.LockRing.GetPrimary(l.key)
+			if newPrimary != l.primary {
+				changed++
+			}
+			break
+		}
+	}
+
+	// Only locks from filer2 should have changed
+	assert.Equal(t, countBefore["filer2:8888"], changed,
+		"only locks from removed server should change primary")
+}
+
+func TestDLM_NodeDropAndJoin_OwnershipDisruption(t *testing.T) {
+	// Scenario: 3 nodes, acquire locks, one drops and a NEW node joins quickly.
+	// The new node steals hash ranges from surviving nodes, not just from the
+	// departed node. This test measures the disruption.
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire many locks
+	numLocks := 100
+	type lockInfo struct {
+		key, token string
+		primary    pb.ServerAddress
+	}
+	locks := make([]lockInfo, numLocks)
+	for i := range locks {
+		key := fmt.Sprintf("churn-lock-%d", i)
+		token, _, primary, err := cluster.acquireLock(key, "owner", 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockInfo{key: key, token: token, primary: primary}
+	}
+
+	// Record primary for each lock before the change
+	beforePrimary := make(map[string]pb.ServerAddress)
+	for _, l := range locks {
+		beforePrimary[l.key] = l.primary
+	}
+
+	// Drop filer3 and immediately add filer4
+	cluster.removeNode("filer3:8888")
+
+	// Promote backups on remaining nodes (simulates OnDlmChangeSnapshot)
+	for _, dlm := range cluster.getNodes() {
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p == dlm.Host && lock.IsBackup {
+				dlm.PromoteLock(lock.Key)
+			}
+		}
+		// Re-replicate primary locks to new backups
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p == dlm.Host && !lock.IsBackup {
+				dlm.replicateToBackup(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq, false)
+			}
+		}
+	}
+	time.Sleep(10 * time.Millisecond)
+
+	// Now add filer4 (new node, empty)
+	cluster.addNode("filer4:8888")
+	time.Sleep(10 * time.Millisecond)
+
+	// Simulate OnDlmChangeSnapshot on all nodes after filer4 joins:
+	// transfer locks that now belong to filer4
+	for host, dlm := range cluster.getNodes() {
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p != host && !lock.IsBackup {
+				// This lock should move to the new primary
+				target := cluster.get(p)
+				if target != nil {
+					target.InsertLock(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq)
+					dlm.DemoteLock(lock.Key)
+				}
+			}
+		}
+	}
+	time.Sleep(10 * time.Millisecond)
+
+	// Count disruptions: locks whose primary changed to a node other than filer3's successor
+	disruptedFromSurvivors := 0
+	disruptedFromDeparted := 0
+	movedToFiler4 := 0
+	for _, l := range locks {
+		// What's the new primary?
+		var newPrimary pb.ServerAddress
+		for _, dlm := range cluster.getNodes() {
+			newPrimary = dlm.LockRing.GetPrimary(l.key)
+			break
+		}
+		oldPrimary := beforePrimary[l.key]
+		if newPrimary != oldPrimary {
+			if oldPrimary == "filer3:8888" {
+				disruptedFromDeparted++
+			} else {
+				disruptedFromSurvivors++
+			}
+		}
+		if newPrimary == "filer4:8888" {
+			movedToFiler4++
+		}
+	}
+
+	t.Logf("Locks disrupted from departed filer3: %d / %d", disruptedFromDeparted, numLocks)
+	t.Logf("Locks disrupted from surviving filer1/filer2: %d / %d", disruptedFromSurvivors, numLocks)
+	t.Logf("Locks now on new filer4: %d / %d", movedToFiler4, numLocks)
+
+	// The key concern: filer4 joining disrupts locks on surviving nodes
+	// With consistent hashing, new node steals ~1/N of each surviving node's keys
+	// Verify that the transfer logic above moved those locks to filer4
+	for _, l := range locks {
+		var newPrimary pb.ServerAddress
+		for _, dlm := range cluster.getNodes() {
+			newPrimary = dlm.LockRing.GetPrimary(l.key)
+			break
+		}
+		target := cluster.get(newPrimary)
+		require.NotNil(t, target, "primary %s should exist", newPrimary)
+
+		lock, found := target.GetLock(l.key)
+		if !found {
+			// Lock may have only a backup copy if transfer happened but
+			// the lock was on the departed node and wasn't re-replicated.
+			// Check all nodes for any copy.
+			anyFound := false
+			for _, dlm := range cluster.getNodes() {
+				if _, f := dlm.GetLock(l.key); f {
+					anyFound = true
+					break
+				}
+			}
+			if !anyFound {
+				t.Errorf("lock %s completely lost (primary should be %s)", l.key, newPrimary)
+			}
+			continue
+		}
+		assert.False(t, lock.IsBackup, "lock %s on primary %s should not be a backup", l.key, newPrimary)
+	}
+}
+
+func TestDLM_RenewalDuringTransferWindow(t *testing.T) {
+	// When a new node joins and steals a key range from a surviving node,
+	// there's a window between ring update and lock transfer. During this
+	// window, a client renewal should still succeed on the old primary
+	// (because it still holds the lock locally).
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Find a key that will move primary when filer4 is added.
+	// Try candidate keys until we find one whose primary changes.
+	var key, renewToken string
+	var primaryHost pb.ServerAddress
+	for i := 0; i < 1000; i++ {
+		candidate := fmt.Sprintf("transfer-window-lock-%d", i)
+		token, _, primary, err := cluster.acquireLock(candidate, "owner1", 30*time.Second)
+		require.NoError(t, err)
+
+		// Check if adding filer4 would move this key's primary
+		tmpRing := NewHashRing(DefaultVnodeCount)
+		tmpRing.SetServers([]pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888", "filer4:8888"})
+		newPrimary := tmpRing.GetPrimary(candidate)
+		if newPrimary != primary {
+			key = candidate
+			renewToken = token
+			primaryHost = primary
+			break
+		}
+	}
+	require.NotEmpty(t, key, "should find a key that moves primary when filer4 joins")
+
+	// Add filer4 — this changes the primary for our key per the ring
+	cluster.addNode("filer4:8888")
+	time.Sleep(10 * time.Millisecond)
+
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	require.NotEqual(t, primaryHost, newPrimary, "key should have moved to a different primary")
+
+	// Renewal on the OLD primary should still succeed because it holds the lock locally
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err, "renewal on old primary should succeed during transfer window")
+	assert.NotEmpty(t, newToken, "should get a new token from old primary")
+	t.Logf("Key %s: primary changed from %s to %s, but renewal on old primary succeeded", key, primaryHost, newPrimary)
+}
+
+func TestDLM_StaleReplicationRejected(t *testing.T) {
+	// Verify that a stale replication (lower seq) does not overwrite a newer one
+	lm := NewLockManager()
+
+	// Insert backup with seq=3
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 1, 3)
+	lock, found := lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(3), lock.Seq)
+
+	// Try to overwrite with stale seq=2 — should be rejected
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 1, 2)
+	lock, found = lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-new", lock.Token, "stale replication should be rejected")
+	assert.Equal(t, int64(3), lock.Seq)
+
+	// Update with higher seq=4 — should succeed
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-newer", "owner1", 1, 4)
+	lock, found = lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-newer", lock.Token, "newer replication should be accepted")
+	assert.Equal(t, int64(4), lock.Seq)
+
+	// Stale unlock (seq=2) should not delete the lock
+	removed := lm.RemoveBackupLockIfSeq("key1", 1, 2)
+	assert.False(t, removed, "stale unlock should be rejected")
+	_, found = lm.GetLock("key1")
+	assert.True(t, found, "lock should still exist after stale unlock")
+
+	// Valid unlock (seq=5) should delete
+	removed = lm.RemoveBackupLockIfSeq("key1", 1, 5)
+	assert.True(t, removed, "valid unlock should be accepted")
+	_, found = lm.GetLock("key1")
+	assert.False(t, found, "lock should be removed after valid unlock")
+}

weed/cluster/lock_manager/hash_ring.go

@@ -0,0 +1,198 @@
+package lock_manager
+
+import (
+	"hash/crc32"
+	"sort"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+const DefaultVnodeCount = 50
+
+// HashRing implements consistent hashing with virtual nodes.
+// When a server is removed, only the keys that hashed to that server
+// are remapped (to the next server on the ring), leaving all other
+// key-to-server mappings stable.
+//
+// UPGRADE NOTE: This replaces the previous modulo-based hashing
+// (hash % len(servers)). The two schemes compute different primaries
+// for the same key, so all filer nodes in the cluster must be upgraded
+// together (or via a rolling restart that completes within the lock TTL
+// window of ~7 seconds) to avoid routing disagreements.
+type HashRing struct {
+	mu            sync.RWMutex
+	vnodeCount    int
+	sortedHashes  []uint32                     // sorted ring positions
+	vnodeToServer map[uint32]pb.ServerAddress   // ring position → server
+	servers       map[pb.ServerAddress]struct{} // set of all servers
+}
+
+func NewHashRing(vnodeCount int) *HashRing {
+	if vnodeCount <= 0 {
+		vnodeCount = DefaultVnodeCount
+	}
+	return &HashRing{
+		vnodeCount:    vnodeCount,
+		vnodeToServer: make(map[uint32]pb.ServerAddress),
+		servers:       make(map[pb.ServerAddress]struct{}),
+	}
+}
+
+// AddServer adds a server with virtual nodes to the ring.
+func (hr *HashRing) AddServer(server pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	if _, exists := hr.servers[server]; exists {
+		return
+	}
+	hr.servers[server] = struct{}{}
+	hr.rebuildRing()
+}
+
+// RemoveServer removes a server and its virtual nodes from the ring.
+func (hr *HashRing) RemoveServer(server pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	if _, exists := hr.servers[server]; !exists {
+		return
+	}
+	delete(hr.servers, server)
+	hr.rebuildRing()
+}
+
+// SetServers replaces the entire server set.
+func (hr *HashRing) SetServers(servers []pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	hr.servers = make(map[pb.ServerAddress]struct{}, len(servers))
+	for _, s := range servers {
+		hr.servers[s] = struct{}{}
+	}
+	hr.rebuildRing()
+}
+
+// GetPrimaryAndBackup returns the primary server for a key and its backup
+// (the next distinct server clockwise on the ring).
+// If there is only one server, backup is empty.
+func (hr *HashRing) GetPrimaryAndBackup(key string) (primary, backup pb.ServerAddress) {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	if len(hr.sortedHashes) == 0 {
+		return "", ""
+	}
+
+	hash := hashKey(key)
+	idx := hr.search(hash)
+	primary = hr.vnodeToServer[hr.sortedHashes[idx]]
+
+	// Walk clockwise to find a different server for backup
+	ringLen := len(hr.sortedHashes)
+	for i := 1; i < ringLen; i++ {
+		candidate := hr.vnodeToServer[hr.sortedHashes[(idx+i)%ringLen]]
+		if candidate != primary {
+			backup = candidate
+			return
+		}
+	}
+	// Only one server — no backup
+	return primary, ""
+}
+
+// GetPrimary returns just the primary server for a key.
+func (hr *HashRing) GetPrimary(key string) pb.ServerAddress {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	if len(hr.sortedHashes) == 0 {
+		return ""
+	}
+
+	hash := hashKey(key)
+	idx := hr.search(hash)
+	return hr.vnodeToServer[hr.sortedHashes[idx]]
+}
+
+// GetServers returns a sorted copy of all servers in the ring.
+func (hr *HashRing) GetServers() []pb.ServerAddress {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	servers := make([]pb.ServerAddress, 0, len(hr.servers))
+	for s := range hr.servers {
+		servers = append(servers, s)
+	}
+	sort.Slice(servers, func(i, j int) bool {
+		return servers[i] < servers[j]
+	})
+	return servers
+}
+
+// ServerCount returns the number of servers in the ring.
+func (hr *HashRing) ServerCount() int {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+	return len(hr.servers)
+}
+
+// rebuildRing rebuilds the sorted hash ring from the current server set.
+// Caller must hold hr.mu write lock.
+func (hr *HashRing) rebuildRing() {
+	hr.vnodeToServer = make(map[uint32]pb.ServerAddress, len(hr.servers)*hr.vnodeCount)
+	hr.sortedHashes = make([]uint32, 0, len(hr.servers)*hr.vnodeCount)
+
+	for server := range hr.servers {
+		for i := 0; i < hr.vnodeCount; i++ {
+			vnodeKey := vnodeKeyFor(server, i)
+			hash := hashKey(vnodeKey)
+			hr.vnodeToServer[hash] = server
+			hr.sortedHashes = append(hr.sortedHashes, hash)
+		}
+	}
+	sort.Slice(hr.sortedHashes, func(i, j int) bool {
+		return hr.sortedHashes[i] < hr.sortedHashes[j]
+	})
+}
+
+// search finds the first ring position >= hash.
+func (hr *HashRing) search(hash uint32) int {
+	idx := sort.Search(len(hr.sortedHashes), func(i int) bool {
+		return hr.sortedHashes[i] >= hash
+	})
+	if idx >= len(hr.sortedHashes) {
+		idx = 0 // wrap around
+	}
+	return idx
+}
+
+func hashKey(key string) uint32 {
+	return crc32.ChecksumIEEE([]byte(key))
+}
+
+func vnodeKeyFor(server pb.ServerAddress, index int) string {
+	// Use a format that distributes well across the ring
+	buf := make([]byte, 0, len(server)+10)
+	buf = append(buf, []byte(server)...)
+	buf = append(buf, '#')
+	buf = appendInt(buf, index)
+	return string(buf)
+}
+
+func appendInt(buf []byte, n int) []byte {
+	if n == 0 {
+		return append(buf, '0')
+	}
+	// Simple int-to-string without importing strconv
+	digits := [20]byte{}
+	pos := len(digits)
+	for n > 0 {
+		pos--
+		digits[pos] = byte('0' + n%10)
+		n /= 10
+	}
+	return append(buf, digits[pos:]...)
+}

weed/cluster/lock_manager/hash_ring_test.go

@@ -0,0 +1,173 @@
+package lock_manager
+
+import (
+	"fmt"
+	"math"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHashRing_BasicOperations(t *testing.T) {
+	hr := NewHashRing(50)
+
+	// Empty ring
+	p, b := hr.GetPrimaryAndBackup("key1")
+	assert.Equal(t, pb.ServerAddress(""), p)
+	assert.Equal(t, pb.ServerAddress(""), b)
+
+	// Single server — no backup
+	hr.AddServer("filer1:8888")
+	p, b = hr.GetPrimaryAndBackup("key1")
+	assert.Equal(t, pb.ServerAddress("filer1:8888"), p)
+	assert.Equal(t, pb.ServerAddress(""), b)
+
+	// Two servers — backup is the other server
+	hr.AddServer("filer2:8888")
+	p, b = hr.GetPrimaryAndBackup("key1")
+	assert.NotEqual(t, p, b)
+	assert.NotEmpty(t, b)
+
+	// Three servers
+	hr.AddServer("filer3:8888")
+	p, b = hr.GetPrimaryAndBackup("key1")
+	assert.NotEqual(t, p, b)
+	assert.NotEmpty(t, b)
+
+	// Remove server
+	hr.RemoveServer("filer2:8888")
+	assert.Equal(t, 2, hr.ServerCount())
+}
+
+func TestHashRing_DuplicateAddRemove(t *testing.T) {
+	hr := NewHashRing(50)
+
+	hr.AddServer("filer1:8888")
+	hr.AddServer("filer1:8888") // duplicate
+	assert.Equal(t, 1, hr.ServerCount())
+
+	hr.RemoveServer("filer1:8888")
+	assert.Equal(t, 0, hr.ServerCount())
+
+	hr.RemoveServer("filer1:8888") // remove non-existent
+	assert.Equal(t, 0, hr.ServerCount())
+}
+
+func TestHashRing_SetServers(t *testing.T) {
+	hr := NewHashRing(50)
+
+	hr.SetServers([]pb.ServerAddress{"a:1", "b:2", "c:3"})
+	assert.Equal(t, 3, hr.ServerCount())
+
+	servers := hr.GetServers()
+	assert.Equal(t, 3, len(servers))
+
+	// SetServers replaces
+	hr.SetServers([]pb.ServerAddress{"x:1", "y:2"})
+	assert.Equal(t, 2, hr.ServerCount())
+}
+
+func TestHashRing_ConsistencyOnRemoval(t *testing.T) {
+	// The key property of consistent hashing: when a server is removed,
+	// only keys that mapped to the removed server change.
+	hr := NewHashRing(50)
+	servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	hr.SetServers(servers)
+
+	numKeys := 1000
+	// Record where each key maps before removal
+	before := make(map[string]pb.ServerAddress, numKeys)
+	for i := 0; i < numKeys; i++ {
+		key := fmt.Sprintf("lock-key-%d", i)
+		before[key] = hr.GetPrimary(key)
+	}
+
+	// Remove filer2
+	hr.RemoveServer("filer2:8888")
+
+	moved := 0
+	for i := 0; i < numKeys; i++ {
+		key := fmt.Sprintf("lock-key-%d", i)
+		after := hr.GetPrimary(key)
+		if before[key] != after {
+			// Only keys from filer2 should move
+			assert.Equal(t, pb.ServerAddress("filer2:8888"), before[key],
+				"key %s moved from %s to %s, but it wasn't on the removed server", key, before[key], after)
+			moved++
+		}
+	}
+	// Roughly 1/3 of keys should move (those that were on filer2)
+	t.Logf("Keys that moved: %d / %d", moved, numKeys)
+	assert.Greater(t, moved, 0, "some keys should have moved")
+	assert.Less(t, moved, numKeys, "not all keys should move")
+}
+
+func TestHashRing_BackupIsSuccessor(t *testing.T) {
+	// After removing primary, the backup should become the new primary
+	hr := NewHashRing(50)
+	servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	hr.SetServers(servers)
+
+	// For each key, verify that removing the primary makes the backup the new primary
+	promoted := 0
+	total := 500
+	for i := 0; i < total; i++ {
+		key := fmt.Sprintf("test-lock-%d", i)
+		primary, backup := hr.GetPrimaryAndBackup(key)
+		assert.NotEqual(t, primary, backup)
+
+		// Temporarily remove primary
+		hr.RemoveServer(primary)
+		newPrimary := hr.GetPrimary(key)
+		if newPrimary == backup {
+			promoted++
+		}
+		// Restore
+		hr.AddServer(primary)
+	}
+	// The backup should become new primary for all keys
+	assert.Equal(t, total, promoted,
+		"backup should become new primary for all keys when primary is removed")
+}
+
+func TestHashRing_Distribution(t *testing.T) {
+	hr := NewHashRing(50)
+	servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	hr.SetServers(servers)
+
+	counts := make(map[pb.ServerAddress]int)
+	numKeys := 3000
+	for i := 0; i < numKeys; i++ {
+		key := fmt.Sprintf("dist-key-%d", i)
+		p := hr.GetPrimary(key)
+		counts[p]++
+	}
+
+	expected := float64(numKeys) / float64(len(servers))
+	for server, count := range counts {
+		deviation := math.Abs(float64(count)-expected) / expected
+		t.Logf("Server %s: %d keys (%.1f%% deviation)", server, count, deviation*100)
+		// Allow up to 40% deviation with 50 vnodes and 3 servers
+		assert.Less(t, deviation, 0.40,
+			"server %s has too many or too few keys: %d (expected ~%d)", server, count, int(expected))
+	}
+}
+
+func TestHashRing_GetPrimary(t *testing.T) {
+	hr := NewHashRing(50)
+
+	// Empty ring
+	assert.Equal(t, pb.ServerAddress(""), hr.GetPrimary("key"))
+
+	hr.SetServers([]pb.ServerAddress{"a:1", "b:2"})
+
+	// Deterministic: same key always maps to same server
+	p1 := hr.GetPrimary("mykey")
+	p2 := hr.GetPrimary("mykey")
+	assert.Equal(t, p1, p2)
+
+	// GetPrimary matches the primary from GetPrimaryAndBackup
+	primary, _ := hr.GetPrimaryAndBackup("mykey")
+	assert.Equal(t, primary, hr.GetPrimary("mykey"))
+}

weed/cluster/lock_manager/lock_manager.go

@@ -3,6 +3,7 @@ package lock_manager
 import (
 	"fmt"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
@@ -17,14 +18,18 @@ var LockNotFound = fmt.Errorf("lock not found")
 
 // LockManager local lock manager, used by distributed lock manager
 type LockManager struct {
-	locks      map[string]*Lock
-	accessLock sync.RWMutex
+	locks            map[string]*Lock
+	accessLock       sync.RWMutex
+	nextGeneration   atomic.Int64
 }
 type Lock struct {
 	Token       string
 	ExpiredAtNs int64
 	Key         string // only used for moving locks
 	Owner       string
+	IsBackup    bool   // true if this node holds the lock as a backup
+	Generation  int64  // monotonic fencing token, increments on fresh acquisition
+	Seq         int64  // per-lock sequence number, increments on every mutation (acquire/renew/unlock)
 }
 
 func NewLockManager() *LockManager {
@@ -35,7 +40,38 @@ func NewLockManager() *LockManager {
 	return t
 }
 
-func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner string) (lockOwner, renewToken string, err error) {
+func (lm *LockManager) NextGeneration() int64 {
+	return lm.nextGeneration.Add(1)
+}
+
+func compareMutationVersion(generation, seq int64, existingGeneration, existingSeq int64) int {
+	switch {
+	case generation < existingGeneration:
+		return -1
+	case generation > existingGeneration:
+		return 1
+	case seq < existingSeq:
+		return -1
+	case seq > existingSeq:
+		return 1
+	default:
+		return 0
+	}
+}
+
+func (lm *LockManager) advanceGenerationFloor(generation int64) {
+	for {
+		current := lm.nextGeneration.Load()
+		if generation < current {
+			return
+		}
+		if lm.nextGeneration.CompareAndSwap(current, generation) {
+			return
+		}
+	}
+}
+
+func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner string) (lockOwner, renewToken string, generation int64, seq int64, err error) {
 	lm.accessLock.Lock()
 	defer lm.accessLock.Unlock()
 
@@ -51,8 +87,10 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
 			} else {
 				// new lock
 				renewToken = uuid.New().String()
-				glog.V(4).Infof("key %s new token %v owner %v", path, renewToken, owner)
-				lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
+				generation = lm.NextGeneration()
+				seq = 1
+				glog.V(4).Infof("key %s new token %v owner %v generation %d", path, renewToken, owner, generation)
+				lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
 				return
 			}
 		}
@@ -61,8 +99,10 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
 		if oldValue.Token == token {
 			// token matches, renew the lock
 			renewToken = uuid.New().String()
+			generation = oldValue.Generation // keep same generation on renewal
+			seq = oldValue.Seq + 1
 			glog.V(4).Infof("key %s old token %v owner %v => %v owner %v", path, oldValue.Token, oldValue.Owner, renewToken, owner)
-			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
+			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
 			return
 		} else {
 			if token == "" {
@@ -79,9 +119,11 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
 		glog.V(4).Infof("key %s no lock owner %v", path, owner)
 		if token == "" {
 			// new lock
-			glog.V(4).Infof("key %s new token %v owner %v", path, token, owner)
 			renewToken = uuid.New().String()
-			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
+			generation = lm.NextGeneration()
+			seq = 1
+			glog.V(4).Infof("key %s new token %v owner %v generation %d", path, renewToken, owner, generation)
+			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
 			return
 		} else {
 			glog.V(4).Infof("key %s non-empty token %v owner %v", path, token, owner)
@@ -91,7 +133,7 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
 	}
 }
 
-func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, err error) {
+func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, generation int64, seq int64, err error) {
 	lm.accessLock.Lock()
 	defer lm.accessLock.Unlock()
 
@@ -100,12 +142,16 @@ func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, err e
 		if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < now.UnixNano() {
 			// lock is expired, delete it
 			isUnlocked = true
+			generation = oldValue.Generation
+			seq = oldValue.Seq + 1
 			glog.V(4).Infof("key %s expired at %v", path, time.Unix(0, oldValue.ExpiredAtNs))
 			delete(lm.locks, path)
 			return
 		}
 		if oldValue.Token == token {
 			isUnlocked = true
+			generation = oldValue.Generation
+			seq = oldValue.Seq + 1
 			glog.V(4).Infof("key %s unlocked with %v", path, token)
 			delete(lm.locks, path)
 			return
@@ -140,10 +186,10 @@ func (lm *LockManager) CleanUp() {
 }
 
 // SelectLocks takes out locks by key
-// if keyFn return true, the lock will be taken out
+// if selectFn returns true, the lock will be removed and returned
 func (lm *LockManager) SelectLocks(selectFn func(key string) bool) (locks []*Lock) {
-	lm.accessLock.RLock()
-	defer lm.accessLock.RUnlock()
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
 
 	now := time.Now().UnixNano()
 
@@ -163,12 +209,90 @@ func (lm *LockManager) SelectLocks(selectFn func(key string) bool) (locks []*Loc
 	return
 }
 
-// InsertLock inserts a lock unconditionally
-func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string, owner string) {
+// InsertLock inserts a lock from a transfer if it is newer than the current state.
+func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string, owner string, generation int64, seq int64) bool {
 	lm.accessLock.Lock()
 	defer lm.accessLock.Unlock()
 
-	lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner}
+	if existing, found := lm.locks[path]; found {
+		if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) <= 0 {
+			glog.V(4).Infof("lock %s: rejecting stale transfer gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
+			return false
+		}
+	}
+	lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
+	lm.advanceGenerationFloor(generation)
+	return true
+}
+
+// InsertBackupLock inserts or updates a lock as a backup copy.
+// It rejects stale mutations by comparing (generation, seq). If a current primary
+// already exists on this node, newer replicated state refreshes that primary copy
+// without demoting it back to a backup.
+func (lm *LockManager) InsertBackupLock(path string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if existing, found := lm.locks[path]; found {
+		if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) <= 0 {
+			glog.V(4).Infof("backup lock %s: rejecting stale gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
+			return
+		}
+		lm.locks[path] = &Lock{
+			Token:       token,
+			ExpiredAtNs: expiredAtNs,
+			Owner:       owner,
+			IsBackup:    existing.IsBackup,
+			Generation:  generation,
+			Seq:         seq,
+		}
+		lm.advanceGenerationFloor(generation)
+		return
+	}
+	lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner, IsBackup: true, Generation: generation, Seq: seq}
+	lm.advanceGenerationFloor(generation)
+}
+
+// RemoveLock removes a lock by key
+func (lm *LockManager) RemoveLock(path string) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+	delete(lm.locks, path)
+}
+
+// RemoveBackupLockIfSeq removes the local copy only if the incoming mutation is
+// not older than the current (generation, seq). This prevents a late unlock from
+// deleting a newer reacquired lock whose seq has reset.
+func (lm *LockManager) RemoveBackupLockIfSeq(path string, generation int64, seq int64) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if existing, found := lm.locks[path]; found {
+		if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) >= 0 {
+			delete(lm.locks, path)
+			return true
+		}
+		glog.V(4).Infof("backup lock %s: rejecting stale unlock gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
+		return false
+	}
+	return true
+}
+
+// GetLock returns a copy of the lock for a key, if it exists and is not expired
+func (lm *LockManager) GetLock(key string) (*Lock, bool) {
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
+
+	lock, found := lm.locks[key]
+	if !found {
+		return nil, false
+	}
+	if time.Now().UnixNano() > lock.ExpiredAtNs {
+		return nil, false
+	}
+	// Return a copy
+	cp := *lock
+	return &cp, true
 }
 
 func (lm *LockManager) GetLockOwner(key string) (owner string, err error) {
@@ -176,8 +300,54 @@ func (lm *LockManager) GetLockOwner(key string) (owner string, err error) {
 	defer lm.accessLock.RUnlock()
 
 	if lock, found := lm.locks[key]; found {
+		if time.Now().UnixNano() > lock.ExpiredAtNs {
+			err = LockNotFound
+			return
+		}
 		return lock.Owner, nil
 	}
 	err = LockNotFound
 	return
 }
+
+// AllLocks returns a copy of all non-expired locks
+func (lm *LockManager) AllLocks() []*Lock {
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
+
+	now := time.Now().UnixNano()
+	var result []*Lock
+	for key, lock := range lm.locks {
+		if now > lock.ExpiredAtNs {
+			continue
+		}
+		cp := *lock
+		cp.Key = key
+		result = append(result, &cp)
+	}
+	return result
+}
+
+// PromoteLock changes a backup lock to a primary lock
+func (lm *LockManager) PromoteLock(key string) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if lock, found := lm.locks[key]; found && lock.IsBackup {
+		lock.IsBackup = false
+		return true
+	}
+	return false
+}
+
+// DemoteLock changes a primary lock to a backup lock
+func (lm *LockManager) DemoteLock(key string) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if lock, found := lm.locks[key]; found && !lock.IsBackup {
+		lock.IsBackup = true
+		return true
+	}
+	return false
+}

weed/cluster/lock_manager/lock_manager_test.go

@@ -0,0 +1,86 @@
+package lock_manager
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLockManager_GetLockOwnerIgnoresExpiredLock(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertLock("key1", time.Now().Add(-time.Second).UnixNano(), "token1", "owner1", 7, 3)
+
+	owner, err := lm.GetLockOwner("key1")
+	assert.Empty(t, owner)
+	assert.ErrorIs(t, err, LockNotFound)
+}
+
+func TestLockManager_InsertLockRejectsStaleTransfer(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 4)
+	lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 7, 3)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(4), lock.Seq)
+}
+
+func TestLockManager_InsertLockAdvancesGenerationCounter(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token1", "owner1", 12, 1)
+
+	_, renewToken, generation, _, err := lm.Lock("key2", time.Now().Add(30*time.Second).UnixNano(), "", "owner2")
+	assert.NoError(t, err)
+	assert.NotEmpty(t, renewToken)
+	assert.Greater(t, generation, int64(12))
+}
+
+func TestLockManager_InsertBackupLockRejectsOlderGeneration(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 1)
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 7, 9)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(1), lock.Seq)
+}
+
+func TestLockManager_InsertBackupLockKeepsPrimaryRole(t *testing.T) {
+	lm := NewLockManager()
+
+	ok := lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 8, 1)
+	assert.True(t, ok)
+
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 2)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.False(t, lock.IsBackup)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(2), lock.Seq)
+}
+
+func TestLockManager_RemoveBackupLockRejectsOlderGeneration(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 1)
+
+	removed := lm.RemoveBackupLockIfSeq("key1", 7, 9)
+	assert.False(t, removed)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(1), lock.Seq)
+}

weed/cluster/lock_manager/lock_ring.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 type LockRingSnapshot struct {
@@ -18,18 +17,18 @@ type LockRingSnapshot struct {
 type LockRing struct {
 	sync.RWMutex
 	snapshots        []*LockRingSnapshot
-	candidateServers map[pb.ServerAddress]struct{}
-	lastUpdateTime   time.Time
 	lastCompactTime  time.Time
 	snapshotInterval time.Duration
 	onTakeSnapshot   func(snapshot []pb.ServerAddress)
 	cleanupWg        sync.WaitGroup
+	Ring             *HashRing // consistent hash ring
+	version          int64    // monotonic version from master, rejects stale updates
 }
 
 func NewLockRing(snapshotInterval time.Duration) *LockRing {
 	return &LockRing{
 		snapshotInterval: snapshotInterval,
-		candidateServers: make(map[pb.ServerAddress]struct{}),
+		Ring:             NewHashRing(DefaultVnodeCount),
 	}
 }
 
@@ -39,52 +38,26 @@ func (r *LockRing) SetTakeSnapshotCallback(onTakeSnapshot func(snapshot []pb.Ser
 	r.onTakeSnapshot = onTakeSnapshot
 }
 
-// AddServer adds a server to the ring
-// if the previous snapshot passed the snapshot interval, create a new snapshot
-func (r *LockRing) AddServer(server pb.ServerAddress) {
-	glog.V(0).Infof("add server %v", server)
-	r.Lock()
-
-	if _, found := r.candidateServers[server]; found {
-		glog.V(0).Infof("add server: already exists %v", server)
-		r.Unlock()
-		return
-	}
-	r.lastUpdateTime = time.Now()
-	r.candidateServers[server] = struct{}{}
-	r.Unlock()
-
-	r.takeSnapshotWithDelayedCompaction()
-}
-
-func (r *LockRing) RemoveServer(server pb.ServerAddress) {
-	glog.V(0).Infof("remove server %v", server)
-
-	r.Lock()
-
-	if _, found := r.candidateServers[server]; !found {
-		r.Unlock()
-		return
-	}
-	r.lastUpdateTime = time.Now()
-	delete(r.candidateServers, server)
-	r.Unlock()
-
-	r.takeSnapshotWithDelayedCompaction()
-}
-
-func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
+// SetSnapshot replaces the ring with a new server list from the master.
+// The version must be >= the current version, otherwise the update is rejected
+// (protects against reordered messages). Version 0 is always accepted (bootstrap).
+func (r *LockRing) SetSnapshot(servers []pb.ServerAddress, version int64) bool {
 
 	sort.Slice(servers, func(i, j int) bool {
 		return servers[i] < servers[j]
 	})
 
 	r.Lock()
-	r.lastUpdateTime = time.Now()
-	// init candidateServers
-	for _, server := range servers {
-		r.candidateServers[server] = struct{}{}
+	if version > 0 && version < r.version {
+		glog.V(0).Infof("LockRing: rejecting stale update v%d (current v%d)", version, r.version)
+		r.Unlock()
+		return false
 	}
+	r.version = version
+	// Update the ring while holding the lock so version and ring state
+	// are always consistent — prevents a concurrent SetSnapshot from
+	// seeing the new version but applying its servers to the old ring.
+	r.Ring.SetServers(servers)
 	r.Unlock()
 
 	r.addOneSnapshot(servers)
@@ -95,23 +68,14 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
 		<-time.After(r.snapshotInterval)
 		r.compactSnapshots()
 	}()
+	return true
 }
 
-func (r *LockRing) takeSnapshotWithDelayedCompaction() {
-	r.doTakeSnapshot()
-
-	r.cleanupWg.Add(1)
-	go func() {
-		defer r.cleanupWg.Done()
-		<-time.After(r.snapshotInterval)
-		r.compactSnapshots()
-	}()
-}
-
-func (r *LockRing) doTakeSnapshot() {
-	servers := r.getSortedServers()
-
-	r.addOneSnapshot(servers)
+// Version returns the current ring version.
+func (r *LockRing) Version() int64 {
+	r.RLock()
+	defer r.RUnlock()
+	return r.version
 }
 
 func (r *LockRing) addOneSnapshot(servers []pb.ServerAddress) {
@@ -138,35 +102,19 @@ func (r *LockRing) compactSnapshots() {
 	r.Lock()
 	defer r.Unlock()
 
-	// Always attempt compaction when called, regardless of lastCompactTime
-	// This ensures proper cleanup even with multiple concurrent compaction requests
-
 	ts := time.Now()
-	// remove old snapshots
 	recentSnapshotIndex := 1
 	for ; recentSnapshotIndex < len(r.snapshots); recentSnapshotIndex++ {
 		if ts.Sub(r.snapshots[recentSnapshotIndex].ts) > r.snapshotInterval {
 			break
 		}
 	}
-	// keep the one that has been running for a while
 	if recentSnapshotIndex+1 <= len(r.snapshots) {
 		r.snapshots = r.snapshots[:recentSnapshotIndex+1]
 	}
 	r.lastCompactTime = ts
 }
 
-func (r *LockRing) getSortedServers() []pb.ServerAddress {
-	sortedServers := make([]pb.ServerAddress, 0, len(r.candidateServers))
-	for server := range r.candidateServers {
-		sortedServers = append(sortedServers, server)
-	}
-	sort.Slice(sortedServers, func(i, j int) bool {
-		return sortedServers[i] < sortedServers[j]
-	})
-	return sortedServers
-}
-
 func (r *LockRing) GetSnapshot() (servers []pb.ServerAddress) {
 	r.RLock()
 	defer r.RUnlock()
@@ -178,7 +126,6 @@ func (r *LockRing) GetSnapshot() (servers []pb.ServerAddress) {
 }
 
 // WaitForCleanup waits for all pending cleanup operations to complete
-// This is useful for testing to ensure deterministic behavior
 func (r *LockRing) WaitForCleanup() {
 	r.cleanupWg.Wait()
 }
@@ -190,14 +137,23 @@ func (r *LockRing) GetSnapshotCount() int {
 	return len(r.snapshots)
 }
 
+// GetPrimaryAndBackup returns the primary and backup servers for a key
+// using the consistent hash ring.
+func (r *LockRing) GetPrimaryAndBackup(key string) (primary, backup pb.ServerAddress) {
+	return r.Ring.GetPrimaryAndBackup(key)
+}
+
+// GetPrimary returns the primary server for a key using the consistent hash ring.
+func (r *LockRing) GetPrimary(key string) pb.ServerAddress {
+	return r.Ring.GetPrimary(key)
+}
+
+// hashKeyToServer uses a temporary consistent hash ring for the given server list.
 func hashKeyToServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
 	if len(servers) == 0 {
 		return ""
 	}
-	x := util.HashStringToLong(key)
-	if x < 0 {
-		x = -x
-	}
-	x = x % int64(len(servers))
-	return servers[x]
+	ring := NewHashRing(DefaultVnodeCount)
+	ring.SetServers(servers)
+	return ring.GetPrimary(key)
 }

weed/cluster/lock_manager/lock_ring_test.go

@@ -8,84 +8,91 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestAddServer(t *testing.T) {
+func TestLockRing_SetSnapshot(t *testing.T) {
 	r := NewLockRing(100 * time.Millisecond)
 
-	// Add servers
-	r.AddServer("localhost:8080")
-	r.AddServer("localhost:8081")
-	r.AddServer("localhost:8082")
-	r.AddServer("localhost:8083")
-	r.AddServer("localhost:8084")
+	// Set 5 servers
+	r.SetSnapshot([]pb.ServerAddress{
+		"localhost:8080", "localhost:8081", "localhost:8082",
+		"localhost:8083", "localhost:8084",
+	}, 1)
 
-	// Verify all servers are present
 	servers := r.GetSnapshot()
 	assert.Equal(t, 5, len(servers))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8080"))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8081"))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8082"))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8083"))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8084"))
 
-	// Remove servers
-	r.RemoveServer("localhost:8084")
-	r.RemoveServer("localhost:8082")
-	r.RemoveServer("localhost:8080")
+	// Replace with 2 servers
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8081", "localhost:8083"}, 2)
 
-	// Wait for all cleanup operations to complete
 	r.WaitForCleanup()
-
-	// Verify only 2 servers remain (localhost:8081 and localhost:8083)
 	servers = r.GetSnapshot()
 	assert.Equal(t, 2, len(servers))
 	assert.Contains(t, servers, pb.ServerAddress("localhost:8081"))
 	assert.Contains(t, servers, pb.ServerAddress("localhost:8083"))
 
-	// Verify cleanup has happened - wait for snapshot interval and check snapshots are compacted
+	// Verify compaction
 	time.Sleep(110 * time.Millisecond)
 	r.WaitForCleanup()
-
-	// Verify snapshot history is cleaned up properly (should have at most 2 snapshots after compaction)
-	snapshotCount := r.GetSnapshotCount()
-	assert.LessOrEqual(t, snapshotCount, 2, "Snapshot history should be compacted")
+	assert.LessOrEqual(t, r.GetSnapshotCount(), 2)
 }
 
-func TestLockRing(t *testing.T) {
+func TestLockRing_SnapshotCompaction(t *testing.T) {
 	r := NewLockRing(100 * time.Millisecond)
 
-	// Test initial snapshot
-	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081"})
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081"}, 1)
 	assert.Equal(t, 1, r.GetSnapshotCount())
-	servers := r.GetSnapshot()
-	assert.Equal(t, 2, len(servers))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8080"))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8081"))
 
-	// Add another server
-	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082"})
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082"}, 2)
 	assert.Equal(t, 2, r.GetSnapshotCount())
-	servers = r.GetSnapshot()
-	assert.Equal(t, 3, len(servers))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8082"))
 
-	// Wait for cleanup interval and add another server
+	// Wait for compaction
 	time.Sleep(110 * time.Millisecond)
 	r.WaitForCleanup()
-	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082", "localhost:8083"})
-	assert.LessOrEqual(t, r.GetSnapshotCount(), 3)
-	servers = r.GetSnapshot()
-	assert.Equal(t, 4, len(servers))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8083"))
 
-	// Wait for cleanup and verify compaction
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082", "localhost:8083"}, 3)
+	assert.LessOrEqual(t, r.GetSnapshotCount(), 3)
+	servers := r.GetSnapshot()
+	assert.Equal(t, 4, len(servers))
+
 	time.Sleep(110 * time.Millisecond)
 	r.WaitForCleanup()
 	assert.LessOrEqual(t, r.GetSnapshotCount(), 2, "Snapshots should be compacted")
 
-	// Add final server
-	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082", "localhost:8083", "localhost:8084"})
+	r.SetSnapshot([]pb.ServerAddress{
+		"localhost:8080", "localhost:8081", "localhost:8082",
+		"localhost:8083", "localhost:8084",
+	}, 4)
 	servers = r.GetSnapshot()
 	assert.Equal(t, 5, len(servers))
-	assert.Contains(t, servers, pb.ServerAddress("localhost:8084"))
-	assert.LessOrEqual(t, r.GetSnapshotCount(), 3)
+}
+
+func TestLockRing_VersionRejectsStale(t *testing.T) {
+	r := NewLockRing(100 * time.Millisecond)
+
+	// Apply version 3
+	ok := r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2", "c:3"}, 3)
+	assert.True(t, ok)
+	assert.Equal(t, int64(3), r.Version())
+	assert.Equal(t, 3, len(r.GetSnapshot()))
+
+	// Stale version 2 — should be rejected
+	ok = r.SetSnapshot([]pb.ServerAddress{"x:1"}, 2)
+	assert.False(t, ok)
+	assert.Equal(t, int64(3), r.Version())
+	assert.Equal(t, 3, len(r.GetSnapshot()), "stale update should not change the ring")
+
+	// Same version 3 — accepted (SetSnapshot accepts version >= current, state-changing)
+	ok = r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2"}, 3)
+	assert.True(t, ok)
+	assert.Equal(t, 2, len(r.GetSnapshot()))
+
+	// Newer version 5 — should be accepted
+	ok = r.SetSnapshot([]pb.ServerAddress{"d:1", "e:2", "f:3", "g:4"}, 5)
+	assert.True(t, ok)
+	assert.Equal(t, int64(5), r.Version())
+	assert.Equal(t, 4, len(r.GetSnapshot()))
+
+	// Version 0 always accepted (bootstrap)
+	ok = r.SetSnapshot([]pb.ServerAddress{"z:1"}, 0)
+	assert.True(t, ok)
+	assert.Equal(t, 1, len(r.GetSnapshot()))
 }

weed/cluster/lock_ring_manager.go

@@ -0,0 +1,164 @@
+package cluster
+
+import (
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+const LockRingStabilizationInterval = 1 * time.Second
+
+// LockRingManager tracks filer membership for the distributed lock ring.
+// It batches rapid topology changes (e.g., node drop + join) with a
+// stabilization timer, then broadcasts the complete member list atomically
+// so filers receive a single consistent ring update instead of multiple
+// intermediate states.
+type LockRingManager struct {
+	mu              sync.Mutex
+	members         map[FilerGroupName]map[pb.ServerAddress]struct{}
+	version         map[FilerGroupName]int64
+	lastBroadcast   map[FilerGroupName]*master_pb.LockRingUpdate
+	pendingTimer    map[FilerGroupName]*time.Timer
+	broadcastFn     func(resp *master_pb.KeepConnectedResponse)
+	stabilizeDelay  time.Duration
+}
+
+func NewLockRingManager(broadcastFn func(resp *master_pb.KeepConnectedResponse)) *LockRingManager {
+	return &LockRingManager{
+		members:        make(map[FilerGroupName]map[pb.ServerAddress]struct{}),
+		version:        make(map[FilerGroupName]int64),
+		lastBroadcast:  make(map[FilerGroupName]*master_pb.LockRingUpdate),
+		pendingTimer:   make(map[FilerGroupName]*time.Timer),
+		broadcastFn:    broadcastFn,
+		stabilizeDelay: LockRingStabilizationInterval,
+	}
+}
+
+// AddServer records a filer joining and schedules a batched broadcast.
+func (lrm *LockRingManager) AddServer(filerGroup FilerGroupName, address pb.ServerAddress) {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+
+	if _, ok := lrm.members[filerGroup]; !ok {
+		lrm.members[filerGroup] = make(map[pb.ServerAddress]struct{})
+	}
+	lrm.members[filerGroup][address] = struct{}{}
+	lrm.scheduleBroadcast(filerGroup)
+}
+
+// RemoveServer records a filer leaving and schedules a batched broadcast.
+func (lrm *LockRingManager) RemoveServer(filerGroup FilerGroupName, address pb.ServerAddress) {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+
+	if members, ok := lrm.members[filerGroup]; ok {
+		delete(members, address)
+	}
+	lrm.scheduleBroadcast(filerGroup)
+}
+
+// GetServers returns the current member list for a filer group.
+func (lrm *LockRingManager) GetServers(filerGroup FilerGroupName) []string {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+
+	members, ok := lrm.members[filerGroup]
+	if !ok {
+		return nil
+	}
+	servers := make([]string, 0, len(members))
+	for addr := range members {
+		servers = append(servers, string(addr))
+	}
+	return servers
+}
+
+// GetVersion returns the current version for a filer group.
+func (lrm *LockRingManager) GetVersion(filerGroup FilerGroupName) int64 {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+	return lrm.version[filerGroup]
+}
+
+// GetLastUpdate returns a copy of the most recently broadcast lock-ring snapshot
+// for the filer group. It intentionally does not expose pending, unstabilized changes.
+func (lrm *LockRingManager) GetLastUpdate(filerGroup FilerGroupName) *master_pb.LockRingUpdate {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+
+	update, ok := lrm.lastBroadcast[filerGroup]
+	if !ok || update == nil {
+		return nil
+	}
+	cp := *update
+	cp.Servers = append([]string(nil), update.Servers...)
+	return &cp
+}
+
+// scheduleBroadcast resets the stabilization timer. If another change arrives
+// before the timer fires, the timer resets, batching the changes.
+// Caller must hold lrm.mu.
+func (lrm *LockRingManager) scheduleBroadcast(filerGroup FilerGroupName) {
+	if timer, ok := lrm.pendingTimer[filerGroup]; ok {
+		if !timer.Stop() {
+			// Timer already fired, callback is running or queued.
+			// It will pick up the latest state from lrm.members, so
+			// just schedule a new one for any further changes.
+		}
+	}
+	lrm.pendingTimer[filerGroup] = time.AfterFunc(lrm.stabilizeDelay, func() {
+		lrm.doBroadcast(filerGroup)
+	})
+}
+
+func (lrm *LockRingManager) doBroadcast(filerGroup FilerGroupName) {
+	lrm.mu.Lock()
+	// Use wall-clock nanoseconds so the version survives master restarts
+	// without persistence — a restarted master produces a version greater
+	// than any pre-restart value (assuming clocks don't jump backward).
+	version := time.Now().UnixNano()
+	lrm.version[filerGroup] = version
+	servers := make([]string, 0)
+	if members, ok := lrm.members[filerGroup]; ok {
+		for addr := range members {
+			servers = append(servers, string(addr))
+		}
+	}
+	update := &master_pb.LockRingUpdate{
+		FilerGroup: string(filerGroup),
+		Servers:    append([]string(nil), servers...),
+		Version:    version,
+	}
+	lrm.lastBroadcast[filerGroup] = update
+	delete(lrm.pendingTimer, filerGroup)
+	lrm.mu.Unlock()
+
+	glog.V(0).Infof("LockRing: broadcasting ring update for group %q version %d: %v", filerGroup, version, servers)
+
+	if lrm.broadcastFn != nil {
+		lrm.broadcastFn(&master_pb.KeepConnectedResponse{
+			LockRingUpdate: update,
+		})
+	}
+}
+
+// FlushPending fires any pending timer immediately (for testing or shutdown).
+func (lrm *LockRingManager) FlushPending(filerGroup FilerGroupName) {
+	lrm.mu.Lock()
+	if timer, ok := lrm.pendingTimer[filerGroup]; ok {
+		if timer.Stop() {
+			// Timer was pending — we stopped it, so we broadcast now
+			delete(lrm.pendingTimer, filerGroup)
+			lrm.mu.Unlock()
+			lrm.doBroadcast(filerGroup)
+		} else {
+			// Timer already fired, callback is running — let it finish
+			lrm.mu.Unlock()
+		}
+	} else {
+		lrm.mu.Unlock()
+	}
+}

weed/cluster/lock_ring_manager_test.go

@@ -0,0 +1,229 @@
+package cluster
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLockRingManager_BatchesRapidChanges(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 100 * time.Millisecond
+
+	group := FilerGroupName("default")
+
+	// Add 3 servers in rapid succession
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+	lrm.AddServer(group, "filer3:8888")
+
+	// No broadcast should have happened yet (timer hasn't fired)
+	mu.Lock()
+	assert.Equal(t, 0, len(broadcasts), "should not broadcast before stabilization delay")
+	mu.Unlock()
+
+	// Wait for stabilization
+	time.Sleep(200 * time.Millisecond)
+
+	mu.Lock()
+	require.Equal(t, 1, len(broadcasts), "should batch into a single broadcast")
+	assert.Equal(t, 3, len(broadcasts[0].Servers), "should include all 3 servers")
+	assert.Greater(t, broadcasts[0].Version, int64(0))
+	mu.Unlock()
+}
+
+func TestLockRingManager_DropAndJoinBatched(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 100 * time.Millisecond
+
+	group := FilerGroupName("default")
+
+	// Set up initial state
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+	lrm.AddServer(group, "filer3:8888")
+	lrm.FlushPending(group)
+
+	mu.Lock()
+	broadcasts = nil // reset
+	mu.Unlock()
+
+	// Simulate drop + join in rapid succession
+	lrm.RemoveServer(group, "filer3:8888")
+	lrm.AddServer(group, "filer4:8888")
+
+	// Should not have broadcast yet
+	mu.Lock()
+	assert.Equal(t, 0, len(broadcasts))
+	mu.Unlock()
+
+	// Wait for stabilization
+	time.Sleep(200 * time.Millisecond)
+
+	mu.Lock()
+	require.Equal(t, 1, len(broadcasts), "drop+join should be batched into single broadcast")
+	servers := broadcasts[0].Servers
+	assert.Equal(t, 3, len(servers), "should have filer1, filer2, filer4")
+	// filer3 should be gone, filer4 should be present
+	serverSet := make(map[string]bool)
+	for _, s := range servers {
+		serverSet[s] = true
+	}
+	assert.False(t, serverSet["filer3:8888"], "filer3 should be removed")
+	assert.True(t, serverSet["filer4:8888"], "filer4 should be added")
+	mu.Unlock()
+}
+
+func TestLockRingManager_VersionIncrements(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 50 * time.Millisecond
+
+	group := FilerGroupName("default")
+
+	lrm.AddServer(group, "filer1:8888")
+	time.Sleep(100 * time.Millisecond)
+
+	lrm.AddServer(group, "filer2:8888")
+	time.Sleep(100 * time.Millisecond)
+
+	mu.Lock()
+	require.Equal(t, 2, len(broadcasts))
+	assert.Greater(t, broadcasts[0].Version, int64(0), "version should be positive")
+	assert.Greater(t, broadcasts[1].Version, broadcasts[0].Version, "versions should be monotonically increasing")
+	mu.Unlock()
+}
+
+func TestLockRingManager_FlushPending(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 10 * time.Second // long delay
+
+	group := FilerGroupName("default")
+
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+
+	// Flush immediately
+	lrm.FlushPending(group)
+
+	mu.Lock()
+	require.Equal(t, 1, len(broadcasts))
+	assert.Equal(t, 2, len(broadcasts[0].Servers))
+	mu.Unlock()
+}
+
+func TestLockRingManager_MultipleGroups(t *testing.T) {
+	var mu sync.Mutex
+	broadcastsByGroup := make(map[string][]*master_pb.LockRingUpdate)
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcastsByGroup[resp.LockRingUpdate.FilerGroup] = append(
+				broadcastsByGroup[resp.LockRingUpdate.FilerGroup], resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 50 * time.Millisecond
+
+	lrm.AddServer("group1", "filer1:8888")
+	lrm.AddServer("group2", "filer2:8888")
+
+	time.Sleep(100 * time.Millisecond)
+
+	mu.Lock()
+	assert.Equal(t, 1, len(broadcastsByGroup["group1"]))
+	assert.Equal(t, 1, len(broadcastsByGroup["group2"]))
+	assert.Equal(t, []string{"filer1:8888"}, broadcastsByGroup["group1"][0].Servers)
+	assert.Equal(t, []string{"filer2:8888"}, broadcastsByGroup["group2"][0].Servers)
+	mu.Unlock()
+}
+
+func TestLockRingManager_GetServers(t *testing.T) {
+	lrm := NewLockRingManager(nil)
+
+	group := FilerGroupName("default")
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+
+	servers := lrm.GetServers(group)
+	assert.Equal(t, 2, len(servers))
+
+	// Contains both
+	serverSet := make(map[string]bool)
+	for _, s := range servers {
+		serverSet[s] = true
+	}
+	assert.True(t, serverSet["filer1:8888"])
+	assert.True(t, serverSet["filer2:8888"])
+
+	// Remove one
+	lrm.RemoveServer(group, "filer1:8888")
+	servers = lrm.GetServers(group)
+	assert.Equal(t, 1, len(servers))
+	assert.Equal(t, "filer2:8888", servers[0])
+}
+
+func TestLockRingManager_NoBroadcastWithoutFn(t *testing.T) {
+	// No panic when broadcastFn is nil
+	lrm := NewLockRingManager(nil)
+	lrm.stabilizeDelay = 10 * time.Millisecond
+
+	lrm.AddServer("default", pb.ServerAddress("filer1:8888"))
+	time.Sleep(50 * time.Millisecond) // should not panic
+}
+
+func TestLockRingManager_GetLastUpdateReturnsBroadcastState(t *testing.T) {
+	lrm := NewLockRingManager(nil)
+
+	group := FilerGroupName("default")
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+	lrm.FlushPending(group)
+
+	update := lrm.GetLastUpdate(group)
+	require.NotNil(t, update)
+	assert.Equal(t, "default", update.FilerGroup)
+	assert.ElementsMatch(t, []string{"filer1:8888", "filer2:8888"}, update.Servers)
+	assert.Greater(t, update.Version, int64(0))
+}

weed/command/filer.go

@@ -421,6 +421,7 @@ func (fo *FilerOptions) startFiler() {
 		go grpcS.Serve(grpcLocalL)
 	}
 	go grpcS.Serve(grpcL)
+	pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)
 
 	if runtime.GOOS != "windows" {
 		localSocket := *fo.localSocket

weed/command/filer_remote_gateway_buckets.go

@@ -164,7 +164,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 
 	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification
-		if message.NewEntry != nil {
+		if metadataEventUpdatesDirectory(resp, filer.DirectoryEtcRemote) {
 			// update
 			if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
 				newMappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
@@ -180,8 +180,11 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				}
 				option.remoteConfs[conf.Name] = conf
 			}
-		} else if message.OldEntry != nil {
+		} else if metadataEventRemovesFromDirectory(resp, filer.DirectoryEtcRemote) {
 			// deletion
+			if message.OldEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
+				option.mappings = &remote_pb.RemoteStorageMapping{}
+			}
 			if strings.HasSuffix(message.OldEntry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
 				conf := &remote_pb.RemoteConf{}
 				if err := proto.Unmarshal(message.OldEntry.Content, conf); err != nil {
@@ -196,7 +199,8 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 
 	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification
-		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
+		sourceInEtcRemote, targetInEtcRemote := metadataEventDirectoryMembership(resp, filer.DirectoryEtcRemote)
+		if sourceInEtcRemote || targetInEtcRemote {
 			return handleEtcRemoteChanges(resp)
 		}
 

weed/command/filer_remote_sync_dir.go

@@ -92,34 +92,38 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 
 	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification
-		if message.NewEntry == nil {
-			return nil
-		}
-		if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
-			mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
-			if readErr != nil {
-				return fmt.Errorf("unmarshal mappings: %w", readErr)
+		if metadataEventUpdatesDirectory(resp, filer.DirectoryEtcRemote) {
+			if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
+				mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
+				if readErr != nil {
+					return fmt.Errorf("unmarshal mappings: %w", readErr)
+				}
+				if remoteLoc, found := mappings.Mappings[mountedDir]; found {
+					if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
+						glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
+					}
+				} else {
+					glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
+					os.Exit(0)
+				}
 			}
-			if remoteLoc, found := mappings.Mappings[mountedDir]; found {
-				if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
-					glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
+			if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
+				conf := &remote_pb.RemoteConf{}
+				if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
+					return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
+				}
+				remoteStorage = conf
+				if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil {
+					client = newClient
+				} else {
+					return err
 				}
-			} else {
-				glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
-				os.Exit(0)
 			}
 		}
-		if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
-			conf := &remote_pb.RemoteConf{}
-			if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
-				return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
-			}
-			remoteStorage = conf
-			if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil {
-				client = newClient
-			} else {
-				return err
-			}
+		if metadataEventRemovesFromDirectory(resp, filer.DirectoryEtcRemote) &&
+			message.OldEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
+			glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
+			os.Exit(0)
 		}
 
 		return nil
@@ -127,7 +131,8 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 
 	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification
-		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
+		sourceInEtcRemote, targetInEtcRemote := metadataEventDirectoryMembership(resp, filer.DirectoryEtcRemote)
+		if sourceInEtcRemote || targetInEtcRemote {
 			return handleEtcRemoteChanges(resp)
 		}
 

weed/command/filer_sync.go

@@ -26,37 +26,39 @@ import (
 )
 
 type SyncOptions struct {
-	isActivePassive *bool
-	filerA          *string
-	filerB          *string
-	aPath           *string
-	aExcludePaths   *string
-	bPath           *string
-	bExcludePaths   *string
-	aReplication    *string
-	bReplication    *string
-	aCollection     *string
-	bCollection     *string
-	aTtlSec         *int
-	bTtlSec         *int
-	aDiskType       *string
-	bDiskType       *string
-	aDebug          *bool
-	bDebug          *bool
-	aFromTsMs       *int64
-	bFromTsMs       *int64
-	aProxyByFiler   *bool
-	bProxyByFiler   *bool
-	metricsHttpIp   *string
-	metricsHttpPort *int
+	isActivePassive  *bool
+	filerA           *string
+	filerB           *string
+	aPath            *string
+	aExcludePaths    *string
+	bPath            *string
+	bExcludePaths    *string
+	aReplication     *string
+	bReplication     *string
+	aCollection      *string
+	bCollection      *string
+	aTtlSec          *int
+	bTtlSec          *int
+	aDiskType        *string
+	bDiskType        *string
+	aDebug           *bool
+	bDebug           *bool
+	aFromTsMs        *int64
+	bFromTsMs        *int64
+	aProxyByFiler    *bool
+	bProxyByFiler    *bool
+	metricsHttpIp    *string
+	metricsHttpPort  *int
 	concurrency      *int
 	chunkConcurrency *int
 	aDoDeleteFiles   *bool
-	bDoDeleteFiles  *bool
-	clientId        int32
-	clientEpoch     atomic.Int32
-	debug           *bool
-	debugPort       *int
+	bDoDeleteFiles   *bool
+	aSecurity        *string
+	bSecurity        *string
+	clientId         int32
+	clientEpoch      atomic.Int32
+	debug            *bool
+	debugPort        *int
 }
 
 const (
@@ -113,6 +115,8 @@ func init() {
 	syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port")
 	syncOptions.aDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("a.doDeleteFiles", true, "delete and update files when synchronizing on filer A")
 	syncOptions.bDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("b.doDeleteFiles", true, "delete and update files when synchronizing on filer B")
+	syncOptions.aSecurity = cmdFilerSynchronize.Flag.String("a.security", "", "security.toml file for filer A when clusters use different certificates")
+	syncOptions.bSecurity = cmdFilerSynchronize.Flag.String("b.security", "", "security.toml file for filer B when clusters use different certificates")
 	syncOptions.debug = cmdFilerSynchronize.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
 	syncOptions.debugPort = cmdFilerSynchronize.Flag.Int("debug.port", 6060, "http port for debugging")
 	syncOptions.clientId = util.RandomInt32()
@@ -144,6 +148,22 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	util.LoadSecurityConfiguration()
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
+	// per-filer TLS when clusters use different certificates
+	grpcDialOptionA := grpcDialOption
+	grpcDialOptionB := grpcDialOption
+	if *syncOptions.aSecurity != "" {
+		var err error
+		if grpcDialOptionA, err = security.LoadClientTLSFromFile(*syncOptions.aSecurity, "grpc.client"); err != nil {
+			glog.Fatalf("load security config for filer A: %v", err)
+		}
+	}
+	if *syncOptions.bSecurity != "" {
+		var err error
+		if grpcDialOptionB, err = security.LoadClientTLSFromFile(*syncOptions.bSecurity, "grpc.client"); err != nil {
+			glog.Fatalf("load security config for filer B: %v", err)
+		}
+	}
+
 	grace.SetupProfiling(*syncCpuProfile, *syncMemProfile)
 
 	filerA := pb.ServerAddress(*syncOptions.filerA)
@@ -153,13 +173,13 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	go statsCollect.StartMetricsServer(*syncOptions.metricsHttpIp, *syncOptions.metricsHttpPort)
 
 	// read a filer signature
-	aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOption, filerA)
+	aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOptionA, filerA)
 	if aFilerErr != nil {
 		glog.Errorf("get filer 'a' signature %d error from %s to %s: %v", aFilerSignature, *syncOptions.filerA, *syncOptions.filerB, aFilerErr)
 		return true
 	}
 	// read b filer signature
-	bFilerSignature, bFilerErr := replication.ReadFilerSignature(grpcDialOption, filerB)
+	bFilerSignature, bFilerErr := replication.ReadFilerSignature(grpcDialOptionB, filerB)
 	if bFilerErr != nil {
 		glog.Errorf("get filer 'b' signature %d error from %s to %s: %v", bFilerSignature, *syncOptions.filerA, *syncOptions.filerB, bFilerErr)
 		return true
@@ -189,9 +209,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	go func() {
 		// a->b
 		// set synchronization start timestamp to offset
-		initOffsetError := initOffsetFromTsMs(grpcDialOption, filerB, aFilerSignature, *syncOptions.bFromTsMs, getSignaturePrefixByPath(*syncOptions.aPath))
+		initOffsetError := initOffsetFromTsMs(grpcDialOptionB, filerB, aFilerSignature, *syncOptions.aFromTsMs, getSignaturePrefixByPath(*syncOptions.aPath))
 		if initOffsetError != nil {
-			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.bFromTsMs, *syncOptions.filerA, *syncOptions.filerB, initOffsetError)
+			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.aFromTsMs, *syncOptions.filerA, *syncOptions.filerB, initOffsetError)
 			os.Exit(2)
 		}
 		for {
@@ -199,11 +219,12 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 			err := doSubscribeFilerMetaChanges(
 				syncOptions.clientId,
 				syncOptions.clientEpoch.Load(),
-				grpcDialOption,
+				grpcDialOptionA,
 				filerA,
 				*syncOptions.aPath,
 				util.StringSplit(*syncOptions.aExcludePaths, ","),
 				*syncOptions.aProxyByFiler,
+				grpcDialOptionB,
 				filerB,
 				*syncOptions.bPath,
 				*syncOptions.bReplication,
@@ -228,9 +249,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	if !*syncOptions.isActivePassive {
 		// b->a
 		// set synchronization start timestamp to offset
-		initOffsetError := initOffsetFromTsMs(grpcDialOption, filerA, bFilerSignature, *syncOptions.aFromTsMs, getSignaturePrefixByPath(*syncOptions.bPath))
+		initOffsetError := initOffsetFromTsMs(grpcDialOptionA, filerA, bFilerSignature, *syncOptions.bFromTsMs, getSignaturePrefixByPath(*syncOptions.bPath))
 		if initOffsetError != nil {
-			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.aFromTsMs, *syncOptions.filerB, *syncOptions.filerA, initOffsetError)
+			glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.bFromTsMs, *syncOptions.filerB, *syncOptions.filerA, initOffsetError)
 			os.Exit(2)
 		}
 		go func() {
@@ -239,11 +260,12 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 				err := doSubscribeFilerMetaChanges(
 					syncOptions.clientId,
 					syncOptions.clientEpoch.Load(),
-					grpcDialOption,
+					grpcDialOptionB,
 					filerB,
 					*syncOptions.bPath,
 					util.StringSplit(*syncOptions.bExcludePaths, ","),
 					*syncOptions.bProxyByFiler,
+					grpcDialOptionA,
 					filerA,
 					*syncOptions.aPath,
 					*syncOptions.aReplication,
@@ -285,12 +307,12 @@ func initOffsetFromTsMs(grpcDialOption grpc.DialOption, targetFiler pb.ServerAdd
 	return nil
 }
 
-func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceExcludePaths []string, sourceReadChunkFromFiler bool, targetFiler pb.ServerAddress, targetPath string,
+func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceExcludePaths []string, sourceReadChunkFromFiler bool, targetGrpcDialOption grpc.DialOption, targetFiler pb.ServerAddress, targetPath string,
 	replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool, concurrency int, chunkConcurrency int, doDeleteFiles bool, sourceFilerSignature int32, targetFilerSignature int32, statePtr *atomic.Pointer[syncState]) error {
 
 	// if first time, start from now
 	// if has previously synced, resume from that point of time
-	sourceFilerOffsetTsNs, err := getOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature)
+	sourceFilerOffsetTsNs, err := getOffset(targetGrpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature)
 	if err != nil {
 		return err
 	}
@@ -300,8 +322,9 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 	// create filer sink
 	filerSource := &source.FilerSource{}
 	filerSource.DoInitialize(sourceFiler.ToHttpAddress(), sourceFiler.ToGrpcAddress(), sourcePath, sourceReadChunkFromFiler)
+	filerSource.SetGrpcDialOption(sourceGrpcDialOption)
 	filerSink := &filersink.FilerSink{}
-	filerSink.DoInitialize(targetFiler.ToHttpAddress(), targetFiler.ToGrpcAddress(), targetPath, replicationStr, collection, ttlSec, diskType, grpcDialOption, sinkWriteChunkByFiler)
+	filerSink.DoInitialize(targetFiler.ToHttpAddress(), targetFiler.ToGrpcAddress(), targetPath, replicationStr, collection, ttlSec, diskType, targetGrpcDialOption, sinkWriteChunkByFiler)
 	filerSink.SetChunkConcurrency(chunkConcurrency)
 	filerSink.SetSourceFiler(filerSource)
 
@@ -328,7 +351,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 	if statePtr != nil {
 		statePtr.Store(&syncState{
 			processor:            processor,
-			grpcDialOption:       grpcDialOption,
+			grpcDialOption:       targetGrpcDialOption,
 			targetFiler:          targetFiler,
 			sourcePath:           sourcePath,
 			sourceFilerSignature: sourceFilerSignature,
@@ -351,7 +374,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 		lastLogTsNs = now
 		// collect synchronous offset
 		statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(offsetTsNs))
-		return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
+		return setOffset(targetGrpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
 	})
 
 	prefix := sourcePath
@@ -372,7 +395,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 		EventErrorType:         pb.RetryForeverOnError,
 	}
 
-	return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
+	return pb.FollowMetadata(sourceFiler, sourceGrpcDialOption, metadataFollowOption, processEventFnWithOffset)
 
 }
 
@@ -445,12 +468,17 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification
 
+		// Derive the target (new-side) directory once. MetadataEventTargetDirectory
+		// returns NewParentPath when set, falling back to resp.Directory for
+		// delete events or legacy events with an empty NewParentPath.
+		targetDir := filer_pb.MetadataEventTargetDirectory(resp)
+
 		var sourceOldKey, sourceNewKey util.FullPath
 		if message.OldEntry != nil {
 			sourceOldKey = util.FullPath(resp.Directory).Child(message.OldEntry.Name)
 		}
 		if message.NewEntry != nil {
-			sourceNewKey = util.FullPath(message.NewParentPath).Child(message.NewEntry.Name)
+			sourceNewKey = util.FullPath(targetDir).Child(message.NewEntry.Name)
 		}
 
 		if debug {
@@ -461,19 +489,24 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			return nil
 		}
 
-		if !strings.HasPrefix(resp.Directory+"/", sourcePath) {
+		// For rename events the key/directory is the old (source) path.
+		// Check both old and new directories so cross-boundary renames
+		// are not silently dropped. The downstream old/new key handling
+		// (lines below) already converts these to create or delete.
+		oldDirExcluded := matchesExcludePath(resp.Directory, excludePaths)
+		newDirExcluded := matchesExcludePath(targetDir, excludePaths)
+		oldDirInScope := util.IsEqualOrUnder(resp.Directory, sourcePath) && !oldDirExcluded
+		newDirInScope := message.NewEntry != nil &&
+			util.IsEqualOrUnder(targetDir, sourcePath) &&
+			!newDirExcluded
+		if !oldDirInScope && !newDirInScope {
 			return nil
 		}
-		for _, excludePath := range excludePaths {
-			if strings.HasPrefix(resp.Directory+"/", excludePath) {
-				return nil
-			}
-		}
 		// Compute per-side exclusion so that rename events crossing an
 		// exclude boundary are handled as delete + create rather than
 		// being entirely skipped.
-		oldExcluded := isEntryExcluded(resp.Directory, message.OldEntry, reExcludeFileName, excludeFileNames, excludePathPatterns)
-		newExcluded := isEntryExcluded(message.NewParentPath, message.NewEntry, reExcludeFileName, excludeFileNames, excludePathPatterns)
+		oldExcluded := oldDirExcluded || isEntryExcluded(resp.Directory, message.OldEntry, reExcludeFileName, excludeFileNames, excludePathPatterns)
+		newExcluded := newDirExcluded || isEntryExcluded(targetDir, message.NewEntry, reExcludeFileName, excludeFileNames, excludePathPatterns)
 
 		if oldExcluded && newExcluded {
 			return nil
@@ -495,7 +528,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			if !doDeleteFiles {
 				return nil
 			}
-			if !strings.HasPrefix(string(sourceOldKey), sourcePath) {
+			if !util.IsEqualOrUnder(string(sourceOldKey), sourcePath) {
 				return nil
 			}
 			key := buildKey(dataSink, message, targetPath, sourceOldKey, sourcePath)
@@ -504,7 +537,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 
 		// handle new entries
 		if filer_pb.IsCreate(resp) {
-			if !strings.HasPrefix(string(sourceNewKey), sourcePath) {
+			if !util.IsEqualOrUnder(string(sourceNewKey), sourcePath) {
 				return nil
 			}
 			key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
@@ -521,18 +554,19 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 		}
 
 		// handle updates
-		if strings.HasPrefix(string(sourceOldKey), sourcePath) {
+		if util.IsEqualOrUnder(string(sourceOldKey), sourcePath) {
 			// old key is in the watched directory
-			if strings.HasPrefix(string(sourceNewKey), sourcePath) {
+			if util.IsEqualOrUnder(string(sourceNewKey), sourcePath) {
 				// new key is also in the watched directory
 				if doDeleteFiles {
 					oldKey := util.Join(targetPath, string(sourceOldKey)[len(sourcePath):])
+					var sinkNewParentPath string
 					if strings.HasSuffix(sourcePath, "/") {
-						message.NewParentPath = util.Join(targetPath, message.NewParentPath[len(sourcePath)-1:])
+						sinkNewParentPath = util.Join(targetPath, targetDir[len(sourcePath)-1:])
 					} else {
-						message.NewParentPath = util.Join(targetPath, message.NewParentPath[len(sourcePath):])
+						sinkNewParentPath = util.Join(targetPath, targetDir[len(sourcePath):])
 					}
-					foundExisting, err := dataSink.UpdateEntry(string(oldKey), message.OldEntry, message.NewParentPath, message.NewEntry, message.DeleteChunks, message.Signatures)
+					foundExisting, err := dataSink.UpdateEntry(string(oldKey), message.OldEntry, sinkNewParentPath, message.NewEntry, message.DeleteChunks, message.Signatures)
 					if foundExisting {
 						return err
 					}
@@ -559,7 +593,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			}
 		} else {
 			// old key is outside the watched directory
-			if strings.HasPrefix(string(sourceNewKey), sourcePath) {
+			if util.IsEqualOrUnder(string(sourceNewKey), sourcePath) {
 				// new key is in the watched directory
 				key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
 				if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil {
@@ -623,6 +657,15 @@ func isEntryExcluded(dir string, entry *filer_pb.Entry, reExcludeFileName *regex
 	return false
 }
 
+func matchesExcludePath(dir string, excludePaths []string) bool {
+	for _, excludePath := range excludePaths {
+		if util.IsEqualOrUnder(dir, excludePath) {
+			return true
+		}
+	}
+	return false
+}
+
 // compileExcludePattern compiles a regexp pattern string, returning nil if empty.
 func compileExcludePattern(pattern string, label string) (*regexp.Regexp, error) {
 	if pattern == "" {

weed/command/filer_sync_process_test.go

@@ -0,0 +1,121 @@
+package command
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var _ sink.ReplicationSink = (*recordingSyncSink)(nil)
+
+type recordingSyncSink struct {
+	deleteKeys []string
+	createKeys []string
+	updateKeys []string
+}
+
+func (s *recordingSyncSink) GetName() string { return "recording" }
+func (s *recordingSyncSink) Initialize(util.Configuration, string) error {
+	return nil
+}
+func (s *recordingSyncSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
+	s.deleteKeys = append(s.deleteKeys, key)
+	return nil
+}
+func (s *recordingSyncSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
+	s.createKeys = append(s.createKeys, key)
+	return nil
+}
+func (s *recordingSyncSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (bool, error) {
+	s.updateKeys = append(s.updateKeys, key)
+	return true, nil
+}
+func (s *recordingSyncSink) GetSinkToDirectory() string         { return "/dest" }
+func (s *recordingSyncSink) SetSourceFiler(*source.FilerSource) {}
+func (s *recordingSyncSink) IsIncremental() bool                { return false }
+
+func TestPathIsEqualOrUnderUsesDirectoryBoundaries(t *testing.T) {
+	tests := []struct {
+		name      string
+		candidate string
+		other     string
+		expected  bool
+	}{
+		{name: "equal", candidate: "/foo", other: "/foo", expected: true},
+		{name: "descendant", candidate: "/foo/bar", other: "/foo", expected: true},
+		{name: "sibling prefix", candidate: "/foobar/bar", other: "/foo", expected: false},
+		{name: "root", candidate: "/foo/bar", other: "/", expected: true},
+		{name: "empty", candidate: "", other: "/foo", expected: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := util.IsEqualOrUnder(tt.candidate, tt.other); got != tt.expected {
+				t.Fatalf("IsEqualOrUnder(%q, %q) = %v, want %v", tt.candidate, tt.other, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestMatchesExcludePathUsesDirectoryBoundaries(t *testing.T) {
+	if !matchesExcludePath("/tmp", []string{"/tmp"}) {
+		t.Fatal("expected exact directory match to be excluded")
+	}
+	if !matchesExcludePath("/tmp/sub", []string{"/tmp"}) {
+		t.Fatal("expected descendant directory to be excluded")
+	}
+	if matchesExcludePath("/tmp2/sub", []string{"/tmp"}) {
+		t.Fatal("did not expect sibling directory to be excluded")
+	}
+}
+
+func TestGenProcessFunctionRenameToSiblingPrefixBecomesDelete(t *testing.T) {
+	dataSink := &recordingSyncSink{}
+	processFn := genProcessFunction("/foo", "/dest", nil, nil, nil, nil, dataSink, true, false)
+
+	err := processFn(&filer_pb.SubscribeMetadataResponse{
+		Directory: "/foo/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: "file.txt"},
+			NewEntry:      &filer_pb.Entry{Name: "file.txt"},
+			NewParentPath: "/foobar/dir",
+		},
+	})
+	if err != nil {
+		t.Fatalf("processFn rename to sibling prefix: %v", err)
+	}
+
+	if len(dataSink.deleteKeys) != 1 || dataSink.deleteKeys[0] != "/dest/dir/file.txt" {
+		t.Fatalf("delete keys = %v, want [/dest/dir/file.txt]", dataSink.deleteKeys)
+	}
+	if len(dataSink.createKeys) != 0 || len(dataSink.updateKeys) != 0 {
+		t.Fatalf("unexpected create/update calls: creates=%v updates=%v", dataSink.createKeys, dataSink.updateKeys)
+	}
+}
+
+func TestGenProcessFunctionRenameFromExcludedDirBecomesCreate(t *testing.T) {
+	dataSink := &recordingSyncSink{}
+	processFn := genProcessFunction("/foo", "/dest", []string{"/foo/excluded"}, nil, nil, nil, dataSink, true, false)
+
+	err := processFn(&filer_pb.SubscribeMetadataResponse{
+		Directory: "/foo/excluded",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: "file.txt"},
+			NewEntry:      &filer_pb.Entry{Name: "file.txt"},
+			NewParentPath: "/foo/live",
+		},
+	})
+	if err != nil {
+		t.Fatalf("processFn rename from excluded dir: %v", err)
+	}
+
+	if len(dataSink.createKeys) != 1 || dataSink.createKeys[0] != "/dest/live/file.txt" {
+		t.Fatalf("create keys = %v, want [/dest/live/file.txt]", dataSink.createKeys)
+	}
+	if len(dataSink.deleteKeys) != 0 || len(dataSink.updateKeys) != 0 {
+		t.Fatalf("unexpected delete/update calls: deletes=%v updates=%v", dataSink.deleteKeys, dataSink.updateKeys)
+	}
+}

weed/command/master.go

@@ -254,6 +254,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		go grpcS.Serve(grpcLocalL)
 	}
 	go grpcS.Serve(grpcL)
+	pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)
 
 	// For multi-master mode with non-Hashicorp raft, wait and check if we should join
 	if !*masterOption.raftHashicorp && !isSingleMaster {

weed/command/metadata_event_scope.go

@@ -0,0 +1,36 @@
+package command
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func metadataEventDirectoryMembership(resp *filer_pb.SubscribeMetadataResponse, dir string) (sourceInDir, targetInDir bool) {
+	if resp == nil || resp.EventNotification == nil {
+		return false, false
+	}
+
+	sourceInDir = util.IsEqualOrUnder(resp.Directory, dir)
+	targetInDir = resp.EventNotification.NewEntry != nil &&
+		util.IsEqualOrUnder(filer_pb.MetadataEventTargetDirectory(resp), dir)
+
+	return sourceInDir, targetInDir
+}
+
+func metadataEventUpdatesDirectory(resp *filer_pb.SubscribeMetadataResponse, dir string) bool {
+	if resp == nil || resp.EventNotification == nil || resp.EventNotification.NewEntry == nil {
+		return false
+	}
+
+	_, targetInDir := metadataEventDirectoryMembership(resp, dir)
+	return targetInDir
+}
+
+func metadataEventRemovesFromDirectory(resp *filer_pb.SubscribeMetadataResponse, dir string) bool {
+	if resp == nil || resp.EventNotification == nil || resp.EventNotification.OldEntry == nil {
+		return false
+	}
+
+	sourceInDir, targetInDir := metadataEventDirectoryMembership(resp, dir)
+	return sourceInDir && !targetInDir
+}

weed/command/metadata_event_scope_test.go

@@ -0,0 +1,116 @@
+package command
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestMetadataEventDirectoryMembershipUsesDirectoryBoundaries(t *testing.T) {
+	resp := &filer_pb.SubscribeMetadataResponse{
+		Directory: filer.DirectoryEtcRemote,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: "remote.conf"},
+			NewEntry:      &filer_pb.Entry{Name: "remote.conf"},
+			NewParentPath: "/etc/remote-sibling",
+		},
+	}
+
+	sourceInDir, targetInDir := metadataEventDirectoryMembership(resp, filer.DirectoryEtcRemote)
+	if !sourceInDir {
+		t.Fatal("expected source directory to match")
+	}
+	if targetInDir {
+		t.Fatal("did not expect sibling target directory to match")
+	}
+}
+
+func TestMetadataEventUpdatesAndRemovesDirectory(t *testing.T) {
+	tests := []struct {
+		name        string
+		resp        *filer_pb.SubscribeMetadataResponse
+		wantUpdate  bool
+		wantRemoval bool
+	}{
+		{
+			name:        "nil response",
+			resp:        nil,
+			wantUpdate:  false,
+			wantRemoval: false,
+		},
+		{
+			name: "create event",
+			resp: &filer_pb.SubscribeMetadataResponse{
+				Directory: filer.DirectoryEtcRemote,
+				EventNotification: &filer_pb.EventNotification{
+					NewEntry:      &filer_pb.Entry{Name: "new.conf"},
+					NewParentPath: filer.DirectoryEtcRemote,
+				},
+			},
+			wantUpdate:  true,
+			wantRemoval: false,
+		},
+		{
+			name: "rename out",
+			resp: &filer_pb.SubscribeMetadataResponse{
+				Directory: filer.DirectoryEtcRemote,
+				EventNotification: &filer_pb.EventNotification{
+					OldEntry:      &filer_pb.Entry{Name: "remote.conf"},
+					NewEntry:      &filer_pb.Entry{Name: "remote.conf"},
+					NewParentPath: "/tmp",
+				},
+			},
+			wantUpdate:  false,
+			wantRemoval: true,
+		},
+		{
+			name: "rename into",
+			resp: &filer_pb.SubscribeMetadataResponse{
+				Directory: "/tmp",
+				EventNotification: &filer_pb.EventNotification{
+					OldEntry:      &filer_pb.Entry{Name: "remote.conf"},
+					NewEntry:      &filer_pb.Entry{Name: "remote.conf"},
+					NewParentPath: filer.DirectoryEtcRemote,
+				},
+			},
+			wantUpdate:  true,
+			wantRemoval: false,
+		},
+		{
+			name: "rename within",
+			resp: &filer_pb.SubscribeMetadataResponse{
+				Directory: filer.DirectoryEtcRemote,
+				EventNotification: &filer_pb.EventNotification{
+					OldEntry:      &filer_pb.Entry{Name: "remote.conf"},
+					NewEntry:      &filer_pb.Entry{Name: "renamed.conf"},
+					NewParentPath: filer.DirectoryEtcRemote,
+				},
+			},
+			wantUpdate:  true,
+			wantRemoval: false,
+		},
+		{
+			name: "delete",
+			resp: &filer_pb.SubscribeMetadataResponse{
+				Directory: filer.DirectoryEtcRemote,
+				EventNotification: &filer_pb.EventNotification{
+					OldEntry: &filer_pb.Entry{Name: "remote.conf"},
+				},
+			},
+			wantUpdate:  false,
+			wantRemoval: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := metadataEventUpdatesDirectory(tt.resp, filer.DirectoryEtcRemote); got != tt.wantUpdate {
+				t.Fatalf("metadataEventUpdatesDirectory() = %v, want %v", got, tt.wantUpdate)
+			}
+			if got := metadataEventRemovesFromDirectory(tt.resp, filer.DirectoryEtcRemote); got != tt.wantRemoval {
+				t.Fatalf("metadataEventRemovesFromDirectory() = %v, want %v", got, tt.wantRemoval)
+			}
+		})
+	}
+}

weed/command/mini.go

@@ -823,6 +823,16 @@ func runMini(cmd *Command, args []string) bool {
 	miniS3Options.filer = &filerAddress
 	miniWebDavOptions.filer = &filerAddress
 
+	// Register Unix socket paths for gRPC services so local inter-service
+	// communication goes through Unix sockets instead of TCP.
+	pb.RegisterLocalGrpcSocket(*miniMasterOptions.portGrpc, fmt.Sprintf("/tmp/seaweedfs-master-grpc-%d.sock", *miniMasterOptions.portGrpc))
+	pb.RegisterLocalGrpcSocket(*miniOptions.v.portGrpc, fmt.Sprintf("/tmp/seaweedfs-volume-grpc-%d.sock", *miniOptions.v.portGrpc))
+	pb.RegisterLocalGrpcSocket(*miniFilerOptions.portGrpc, fmt.Sprintf("/tmp/seaweedfs-filer-grpc-%d.sock", *miniFilerOptions.portGrpc))
+	if *miniS3Options.portGrpc > 0 {
+		pb.RegisterLocalGrpcSocket(*miniS3Options.portGrpc, fmt.Sprintf("/tmp/seaweedfs-s3-grpc-%d.sock", *miniS3Options.portGrpc))
+	}
+	pb.RegisterLocalGrpcSocket(*miniAdminOptions.grpcPort, fmt.Sprintf("/tmp/seaweedfs-admin-grpc-%d.sock", *miniAdminOptions.grpcPort))
+
 	go stats_collect.StartMetricsServer(*miniMetricsHttpIp, *miniMetricsHttpPort)
 
 	if *miniMasterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {

weed/command/mount.go

@@ -31,6 +31,7 @@ type MountOptions struct {
 	uidMap             *string
 	gidMap             *string
 	readOnly           *bool
+	includeSystemEntries *bool
 	debug              *bool
 	debugPort          *int
 	localSocket        *string
@@ -99,6 +100,7 @@ func init() {
 	mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
 	mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")
 	mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only")
+	mountOptions.includeSystemEntries = cmdMount.Flag.Bool("includeSystemEntries", false, "show filer system entries (e.g. /topics, /etc) in directory listings")
 	mountOptions.debug = cmdMount.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
 	mountOptions.debugPort = cmdMount.Flag.Int("debug.port", 6061, "http port for debugging")
 	mountOptions.localSocket = cmdMount.Flag.String("localSocket", "", "default to /tmp/seaweedfs-mount-<mount_dir_hash>.sock")

weed/command/mount_std.go

@@ -340,6 +340,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		VolumeServerAccess:          *mountOptions.volumeServerAccess,
 		Cipher:                      cipher,
 		UidGidMapper:                uidGidMapper,
+		IncludeSystemEntries:        *option.includeSystemEntries,
 		DisableXAttr:                *option.disableXAttr,
 		IsMacOs:                     runtime.GOOS == "darwin",
 		MetadataFlushSeconds:        *option.metadataFlushSeconds,

weed/command/s3.go

@@ -376,6 +376,7 @@ func (s3opt *S3Options) startS3Server() bool {
 		go grpcS.Serve(grpcLocalL)
 	}
 	go grpcS.Serve(grpcL)
+	pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)
 
 	if *s3opt.tlsPrivateKey != "" {
 		// Check for port conflict when both HTTP and HTTPS are enabled on the same port

weed/command/scaffold/notification.toml

@@ -22,6 +22,17 @@ hosts = [
 topic = "seaweedfs_filer"
 offsetFile = "./last.offset"
 offsetSaveIntervalSeconds = 10
+# SASL Authentication
+sasl_enabled = false
+sasl_mechanism = "PLAIN"            # PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
+sasl_username = ""
+sasl_password = ""
+# TLS/SSL
+tls_enabled = false
+tls_ca_cert = ""                    # path to CA certificate PEM file
+tls_client_cert = ""                # path to client certificate PEM file (for mTLS)
+tls_client_key = ""                 # path to client private key PEM file (for mTLS)
+tls_insecure_skip_verify = false
 
 
 [notification.aws_sqs]

weed/command/server.go

@@ -330,6 +330,28 @@ func runServer(cmd *Command, args []string) bool {
 	}
 	filerOptions.defaultLevelDbDirectory = masterOptions.metaFolder
 
+	// Register Unix socket paths for gRPC services running in this process
+	// so local inter-service communication uses Unix sockets instead of TCP.
+	// Resolve gRPC ports early (same calculation each service does internally).
+	for _, svc := range []struct {
+		starting *bool
+		portGrpc *int
+		port     *int
+		name     string
+	}{
+		{isStartingMasterServer, masterOptions.portGrpc, masterOptions.port, "master"},
+		{isStartingVolumeServer, serverOptions.v.portGrpc, serverOptions.v.port, "volume"},
+		{isStartingFiler, filerOptions.portGrpc, filerOptions.port, "filer"},
+		{isStartingS3, s3Options.portGrpc, s3Options.port, "s3"},
+	} {
+		if *svc.starting {
+			if *svc.portGrpc == 0 {
+				*svc.portGrpc = 10000 + *svc.port
+			}
+			pb.RegisterLocalGrpcSocket(*svc.portGrpc, fmt.Sprintf("/tmp/seaweedfs-%s-grpc-%d.sock", svc.name, *svc.portGrpc))
+		}
+	}
+
 	serverWhiteList := util.StringSplit(*serverWhiteListOption, ",")
 
 	if *isStartingFiler {

weed/command/volume.go

@@ -415,6 +415,7 @@ func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerSe
 			glog.Fatalf("start gRPC service failed, %s", err)
 		}
 	}()
+	pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)
 	return grpcS
 }
 

weed/filer/empty_folder_cleanup/empty_folder_cleaner.go

@@ -27,6 +27,7 @@ type FilerOperations interface {
 	CountDirectoryEntries(ctx context.Context, dirPath util.FullPath, limit int) (count int, err error)
 	DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32, ifNotModifiedAfter int64) error
 	GetEntryAttributes(ctx context.Context, p util.FullPath) (attributes map[string][]byte, err error)
+	IsDirectoryKeyObject(ctx context.Context, p util.FullPath) (bool, error)
 }
 
 // folderState tracks the state of a folder for empty folder cleanup
@@ -106,24 +107,11 @@ func (efc *EmptyFolderCleaner) IsEnabled() bool {
 
 // ownsFolder checks if this filer owns the folder via consistent hashing
 func (efc *EmptyFolderCleaner) ownsFolder(folder string) bool {
-	servers := efc.lockRing.GetSnapshot()
-	if len(servers) <= 1 {
-		return true // Single filer case
+	primary := efc.lockRing.GetPrimary(folder)
+	if primary == "" {
+		return true // Single filer case or no servers
 	}
-	return efc.hashKeyToServer(folder, servers) == efc.host
-}
-
-// hashKeyToServer uses consistent hashing to map a folder to a server
-func (efc *EmptyFolderCleaner) hashKeyToServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
-	if len(servers) == 0 {
-		return ""
-	}
-	x := util.HashStringToLong(key)
-	if x < 0 {
-		x = -x
-	}
-	x = x % int64(len(servers))
-	return servers[x]
+	return primary == efc.host
 }
 
 // OnDeleteEvent is called when a file or directory is deleted
@@ -312,6 +300,16 @@ func (efc *EmptyFolderCleaner) executeCleanup(folder string, triggeredBy string)
 		return
 	}
 
+	// Skip explicitly created directory markers (e.g., PUT /bucket/folder/)
+	// These have a MIME type set and should be preserved even when empty
+	if isKeyObj, err := efc.filer.IsDirectoryKeyObject(ctx, util.FullPath(folder)); err != nil {
+		glog.V(2).Infof("EmptyFolderCleaner: error checking directory key object %s: %v", folder, err)
+		return
+	} else if isKeyObj {
+		glog.V(3).Infof("EmptyFolderCleaner: skipping %s (triggered by %s), explicit directory marker", folder, triggeredBy)
+		return
+	}
+
 	// Delete the empty folder
 	glog.Infof("EmptyFolderCleaner: deleting empty folder %s (triggered by %s)", folder, triggeredBy)
 	if err := efc.deleteFolder(ctx, folder); err != nil {

weed/filer/empty_folder_cleanup/empty_folder_cleaner_test.go

@@ -12,9 +12,10 @@ import (
 )
 
 type mockFilerOps struct {
-	countFn  func(path util.FullPath) (int, error)
-	deleteFn func(path util.FullPath) error
-	attrsFn  func(path util.FullPath) (map[string][]byte, error)
+	countFn        func(path util.FullPath) (int, error)
+	deleteFn       func(path util.FullPath) error
+	attrsFn        func(path util.FullPath) (map[string][]byte, error)
+	isDirKeyObjFn  func(path util.FullPath) (bool, error)
 }
 
 func (m *mockFilerOps) CountDirectoryEntries(_ context.Context, dirPath util.FullPath, _ int) (int, error) {
@@ -38,6 +39,13 @@ func (m *mockFilerOps) GetEntryAttributes(_ context.Context, p util.FullPath) (m
 	return m.attrsFn(p)
 }
 
+func (m *mockFilerOps) IsDirectoryKeyObject(_ context.Context, p util.FullPath) (bool, error) {
+	if m.isDirKeyObjFn == nil {
+		return false, nil
+	}
+	return m.isDirKeyObjFn(p)
+}
+
 func Test_isUnderPath(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -155,7 +163,7 @@ func TestEmptyFolderCleaner_ownsFolder(t *testing.T) {
 		"filer2:8888",
 		"filer3:8888",
 	}
-	lockRing.SetSnapshot(servers)
+	lockRing.SetSnapshot(servers, 0)
 
 	// Create cleaner for filer1
 	cleaner1 := &EmptyFolderCleaner{
@@ -205,7 +213,7 @@ func TestEmptyFolderCleaner_ownsFolder(t *testing.T) {
 func TestEmptyFolderCleaner_ownsFolder_singleServer(t *testing.T) {
 	// Create a LockRing with a single server
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
 	cleaner := &EmptyFolderCleaner{
 		lockRing: lockRing,
@@ -243,7 +251,7 @@ func TestEmptyFolderCleaner_ownsFolder_emptyRing(t *testing.T) {
 
 func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanup(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
 	cleaner := &EmptyFolderCleaner{
 		lockRing:     lockRing,
@@ -279,7 +287,7 @@ func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanup(t *testing.T) {
 
 func TestEmptyFolderCleaner_OnDeleteEvent_deduplication(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
 	cleaner := &EmptyFolderCleaner{
 		lockRing:     lockRing,
@@ -309,7 +317,7 @@ func TestEmptyFolderCleaner_OnDeleteEvent_deduplication(t *testing.T) {
 
 func TestEmptyFolderCleaner_OnDeleteEvent_multipleFolders(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
 	cleaner := &EmptyFolderCleaner{
 		lockRing:     lockRing,
@@ -338,7 +346,7 @@ func TestEmptyFolderCleaner_OnDeleteEvent_multipleFolders(t *testing.T) {
 
 func TestEmptyFolderCleaner_OnDeleteEvent_notOwner(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888", "filer2:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888", "filer2:8888"}, 0)
 
 	// Create cleaner for filer that doesn't own the folder
 	cleaner := &EmptyFolderCleaner{
@@ -377,7 +385,7 @@ func TestEmptyFolderCleaner_OnDeleteEvent_notOwner(t *testing.T) {
 
 func TestEmptyFolderCleaner_OnDeleteEvent_disabled(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
 	cleaner := &EmptyFolderCleaner{
 		lockRing:     lockRing,
@@ -405,7 +413,7 @@ func TestEmptyFolderCleaner_OnDeleteEvent_disabled(t *testing.T) {
 
 func TestEmptyFolderCleaner_OnDeleteEvent_directoryDeletion(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
 	cleaner := &EmptyFolderCleaner{
 		lockRing:     lockRing,
@@ -434,7 +442,7 @@ func TestEmptyFolderCleaner_OnDeleteEvent_directoryDeletion(t *testing.T) {
 
 func TestEmptyFolderCleaner_cachedCounts(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
 	cleaner := &EmptyFolderCleaner{
 		lockRing:     lockRing,
@@ -483,7 +491,7 @@ func TestEmptyFolderCleaner_cachedCounts(t *testing.T) {
 
 func TestEmptyFolderCleaner_Stop(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
 	cleaner := &EmptyFolderCleaner{
 		lockRing:     lockRing,
@@ -518,7 +526,7 @@ func TestEmptyFolderCleaner_Stop(t *testing.T) {
 
 func TestEmptyFolderCleaner_cacheEviction(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
 	cleaner := &EmptyFolderCleaner{
 		lockRing:     lockRing,
@@ -573,7 +581,7 @@ func TestEmptyFolderCleaner_cacheEviction(t *testing.T) {
 
 func TestEmptyFolderCleaner_cacheEviction_skipsEntriesInQueue(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
 	cleaner := &EmptyFolderCleaner{
 		lockRing:     lockRing,
@@ -607,7 +615,7 @@ func TestEmptyFolderCleaner_cacheEviction_skipsEntriesInQueue(t *testing.T) {
 
 func TestEmptyFolderCleaner_queueFIFOOrder(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
 	cleaner := &EmptyFolderCleaner{
 		lockRing:     lockRing,
@@ -649,7 +657,7 @@ func TestEmptyFolderCleaner_queueFIFOOrder(t *testing.T) {
 
 func TestEmptyFolderCleaner_processCleanupQueue_drainsAllOnceTriggered(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
 	var deleted []string
 	mock := &mockFilerOps{
@@ -693,7 +701,7 @@ func TestEmptyFolderCleaner_processCleanupQueue_drainsAllOnceTriggered(t *testin
 
 func TestEmptyFolderCleaner_executeCleanup_bucketPolicyDisabledSkips(t *testing.T) {
 	lockRing := lock_manager.NewLockRing(5 * time.Second)
-	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
+	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
 
 	var deleted []string
 	mock := &mockFilerOps{
@@ -733,3 +741,70 @@ func TestEmptyFolderCleaner_executeCleanup_bucketPolicyDisabledSkips(t *testing.
 		t.Fatalf("expected folder %s to be skipped, got deletions %v", folder, deleted)
 	}
 }
+
+func TestEmptyFolderCleaner_executeCleanup_directoryMarker(t *testing.T) {
+	testCases := []struct {
+		name           string
+		isDirKeyObj    bool
+		expectDeletion bool
+	}{
+		{
+			name:           "skips explicit directory marker",
+			isDirKeyObj:    true,
+			expectDeletion: false,
+		},
+		{
+			name:           "deletes implicit empty folder",
+			isDirKeyObj:    false,
+			expectDeletion: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			lockRing := lock_manager.NewLockRing(5 * time.Second)
+			lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
+
+			var deleted []string
+			mock := &mockFilerOps{
+				countFn: func(_ util.FullPath) (int, error) {
+					return 0, nil
+				},
+				deleteFn: func(path util.FullPath) error {
+					deleted = append(deleted, string(path))
+					return nil
+				},
+				isDirKeyObjFn: func(path util.FullPath) (bool, error) {
+					return tc.isDirKeyObj, nil
+				},
+			}
+
+			cleaner := &EmptyFolderCleaner{
+				filer:          mock,
+				lockRing:       lockRing,
+				host:           "filer1:8888",
+				bucketPath:     "/buckets",
+				enabled:        true,
+				folderCounts:   make(map[string]*folderState),
+				cleanupQueue:   NewCleanupQueue(1000, time.Minute),
+				maxCountCheck:  1000,
+				cacheExpiry:    time.Minute,
+				processorSleep: time.Second,
+				stopCh:         make(chan struct{}),
+			}
+
+			folder := "/buckets/test/folder"
+			cleaner.executeCleanup(folder, "triggered_item")
+
+			if tc.expectDeletion {
+				if len(deleted) != 1 || deleted[0] != folder {
+					t.Fatalf("expected implicit empty folder %s to be deleted, got deletions %v", folder, deleted)
+				}
+			} else {
+				if len(deleted) != 0 {
+					t.Fatalf("expected explicit directory marker %s to be preserved, got deletions %v", folder, deleted)
+				}
+			}
+		})
+	}
+}

weed/filer/filer.go

@@ -119,7 +119,7 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste
 		address := pb.ServerAddress(node.Address)
 		snapshot = append(snapshot, address)
 	}
-	f.Dlm.LockRing.SetSnapshot(snapshot)
+	f.Dlm.LockRing.SetSnapshot(snapshot, 0)
 	glog.V(0).Infof("%s aggregate from peers %+v", self, snapshot)
 
 	// Initialize the empty folder cleaner using the same LockRing as Dlm for consistent hashing
@@ -130,15 +130,18 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste
 		if update.NodeType != cluster.FilerType {
 			return
 		}
-		address := pb.ServerAddress(update.Address)
-
-		if update.IsAdd {
-			f.Dlm.LockRing.AddServer(address)
-		} else {
-			f.Dlm.LockRing.RemoveServer(address)
-		}
+		// Lock ring is now managed by the master via LockRingUpdate,
+		// so we no longer call AddServer/RemoveServer here.
 		f.MetaAggregator.OnPeerUpdate(update, startFrom)
 	})
+	f.MasterClient.SetOnLockRingUpdateFn(func(update *master_pb.LockRingUpdate) {
+		var servers []pb.ServerAddress
+		for _, s := range update.Servers {
+			servers = append(servers, pb.ServerAddress(s))
+		}
+		glog.V(0).Infof("LockRing: applying master ring update v%d: %v", update.Version, servers)
+		f.Dlm.LockRing.SetSnapshot(servers, update.Version)
+	})
 
 	for _, peerUpdate := range existingNodes {
 		f.MetaAggregator.OnPeerUpdate(peerUpdate, startFrom)
@@ -559,3 +562,17 @@ func (f *Filer) GetEntryAttributes(ctx context.Context, p util.FullPath) (map[st
 	}
 	return entry.Extended, nil
 }
+
+func (f *Filer) IsDirectoryKeyObject(ctx context.Context, p util.FullPath) (bool, error) {
+	entry, err := f.FindEntry(ctx, p)
+	if err != nil {
+		if errors.Is(err, filer_pb.ErrNotFound) {
+			return false, nil
+		}
+		return false, err
+	}
+	if entry == nil {
+		return false, nil
+	}
+	return entry.IsDirectory() && entry.Mime != "", nil
+}

weed/filer/filer_notify.go

@@ -26,6 +26,10 @@ func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry
 }
 
 func (f *Filer) notifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) *filer_pb.SubscribeMetadataResponse {
+	if metadataEventsSuppressed(ctx) {
+		return nil
+	}
+
 	var fullpath string
 	if oldEntry != nil {
 		fullpath = string(oldEntry.FullPath)

weed/filer/filer_on_meta_event.go

@@ -19,17 +19,25 @@ func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse)
 
 func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
 	message := event.EventNotification
+	oldDir := event.Directory
+	newDir := filer_pb.MetadataEventTargetDirectory(event)
 
-	if f.DirBucketsPath == event.Directory {
-		if filer_pb.IsCreate(event) {
-			if message.NewEntry.IsDirectory {
-				f.Store.OnBucketCreation(message.NewEntry.Name)
-			}
+	if filer_pb.IsCreate(event) {
+		if newDir == f.DirBucketsPath && message.NewEntry.IsDirectory {
+			f.Store.OnBucketCreation(message.NewEntry.Name)
 		}
-		if filer_pb.IsDelete(event) {
-			if message.OldEntry.IsDirectory {
-				f.Store.OnBucketDeletion(message.OldEntry.Name)
-			}
+	}
+	if filer_pb.IsDelete(event) {
+		if oldDir == f.DirBucketsPath && message.OldEntry.IsDirectory {
+			f.Store.OnBucketDeletion(message.OldEntry.Name)
+		}
+	}
+	if filer_pb.IsRename(event) {
+		if oldDir == f.DirBucketsPath && message.OldEntry.IsDirectory {
+			f.Store.OnBucketDeletion(message.OldEntry.Name)
+		}
+		if newDir == f.DirBucketsPath && message.NewEntry.IsDirectory {
+			f.Store.OnBucketCreation(message.NewEntry.Name)
 		}
 	}
 }
@@ -72,10 +80,8 @@ func (f *Filer) onEmptyFolderCleanupEvents(event *filer_pb.SubscribeMetadataResp
 }
 
 func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataResponse) {
-	if DirectoryEtcSeaweedFS != event.Directory {
-		if DirectoryEtcSeaweedFS != event.EventNotification.NewParentPath {
-			return
-		}
+	if !filer_pb.MetadataEventTouchesDirectory(event, DirectoryEtcSeaweedFS) {
+		return
 	}
 
 	entry := event.EventNotification.NewEntry

weed/filer/filer_on_meta_event_test.go

@@ -0,0 +1,70 @@
+package filer
+
+import (
+	"context"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type bucketTrackingStore struct {
+	created []string
+	deleted []string
+}
+
+func (s *bucketTrackingStore) GetName() string { return "bucket-tracking" }
+func (s *bucketTrackingStore) Initialize(configuration util.Configuration, prefix string) error {
+	return nil
+}
+func (s *bucketTrackingStore) InsertEntry(context.Context, *Entry) error { return nil }
+func (s *bucketTrackingStore) UpdateEntry(context.Context, *Entry) error { return nil }
+func (s *bucketTrackingStore) FindEntry(context.Context, util.FullPath) (*Entry, error) {
+	return nil, filer_pb.ErrNotFound
+}
+func (s *bucketTrackingStore) DeleteEntry(context.Context, util.FullPath) error          { return nil }
+func (s *bucketTrackingStore) DeleteFolderChildren(context.Context, util.FullPath) error { return nil }
+func (s *bucketTrackingStore) ListDirectoryEntries(context.Context, util.FullPath, string, bool, int64, ListEachEntryFunc) (string, error) {
+	return "", nil
+}
+func (s *bucketTrackingStore) ListDirectoryPrefixedEntries(context.Context, util.FullPath, string, bool, int64, string, ListEachEntryFunc) (string, error) {
+	return "", nil
+}
+func (s *bucketTrackingStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (s *bucketTrackingStore) CommitTransaction(context.Context) error     { return nil }
+func (s *bucketTrackingStore) RollbackTransaction(context.Context) error   { return nil }
+func (s *bucketTrackingStore) KvPut(context.Context, []byte, []byte) error { return nil }
+func (s *bucketTrackingStore) KvGet(context.Context, []byte) ([]byte, error) {
+	return nil, ErrKvNotFound
+}
+func (s *bucketTrackingStore) KvDelete(context.Context, []byte) error { return nil }
+func (s *bucketTrackingStore) Shutdown()                              {}
+func (s *bucketTrackingStore) OnBucketCreation(bucket string)         { s.created = append(s.created, bucket) }
+func (s *bucketTrackingStore) OnBucketDeletion(bucket string)         { s.deleted = append(s.deleted, bucket) }
+func (s *bucketTrackingStore) CanDropWholeBucket() bool               { return false }
+
+func TestOnBucketEventsRenameIntoBucketsRootCreatesBucket(t *testing.T) {
+	store := &bucketTrackingStore{}
+	f := &Filer{
+		DirBucketsPath: "/buckets",
+		Store:          NewFilerStoreWrapper(store),
+	}
+
+	f.onBucketEvents(&filer_pb.SubscribeMetadataResponse{
+		Directory: "/tmp",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: "migrated", IsDirectory: true},
+			NewEntry:      &filer_pb.Entry{Name: "migrated", IsDirectory: true},
+			NewParentPath: "/buckets",
+		},
+	})
+
+	if len(store.created) != 1 || store.created[0] != "migrated" {
+		t.Fatalf("created buckets = %v, want [migrated]", store.created)
+	}
+	if len(store.deleted) != 0 {
+		t.Fatalf("deleted buckets = %v, want []", store.deleted)
+	}
+}

weed/filer/filerstore_wrapper.go

@@ -25,10 +25,12 @@ type VirtualFilerStore interface {
 	FilerStore
 	DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error
 	DeleteOneEntry(ctx context.Context, entry *Entry) error
+	InsertEntryKnownAbsent(ctx context.Context, entry *Entry) error
 	AddPathSpecificStore(path string, storeId string, store FilerStore)
 	OnBucketCreation(bucket string)
 	OnBucketDeletion(bucket string)
 	CanDropWholeBucket() bool
+	SameActualStore(a, b util.FullPath) bool
 }
 
 type FilerStoreWrapper struct {
@@ -106,6 +108,13 @@ func (fsw *FilerStoreWrapper) getActualStore(path util.FullPath) (store FilerSto
 	return
 }
 
+// SameActualStore reports whether two paths resolve to the same underlying
+// store. When path-specific stores are configured, different subtrees may
+// be served by different backends.
+func (fsw *FilerStoreWrapper) SameActualStore(a, b util.FullPath) bool {
+	return fsw.getActualStore(a) == fsw.getActualStore(b)
+}
+
 func (fsw *FilerStoreWrapper) getDefaultStore() (store FilerStore) {
 	return fsw.defaultStore
 }
@@ -143,6 +152,31 @@ func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) err
 	return actualStore.InsertEntry(ctx, entry)
 }
 
+// InsertEntryKnownAbsent skips the pre-insert FindEntry path when the caller has
+// already established that the target path does not exist.
+func (fsw *FilerStoreWrapper) InsertEntryKnownAbsent(ctx context.Context, entry *Entry) error {
+	ctx = context.WithoutCancel(ctx)
+	actualStore := fsw.getActualStore(entry.FullPath)
+	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "insert").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "insert").Observe(time.Since(start).Seconds())
+	}()
+
+	filer_pb.BeforeEntrySerialization(entry.GetChunks())
+	normalizeEntryMimeForStore(entry)
+
+	if len(entry.HardLinkId) > 0 {
+		glog.V(4).InfofCtx(ctx, "InsertEntryKnownAbsent %s has HardLinkId %x counter=%d",
+			entry.FullPath, entry.HardLinkId, entry.HardLinkCounter)
+		if err := fsw.setHardLink(ctx, entry); err != nil {
+			return fmt.Errorf("setHardLink %x: %v", entry.HardLinkId, err)
+		}
+	}
+
+	return actualStore.InsertEntry(ctx, entry)
+}
+
 func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) error {
 	ctx = context.WithoutCancel(ctx)
 	actualStore := fsw.getActualStore(entry.FullPath)

weed/filer/metadata_event_context.go

@@ -0,0 +1,20 @@
+package filer
+
+import "context"
+
+type suppressMetadataEventsKey struct{}
+
+// WithSuppressedMetadataEvents disables automatic metadata event emission for
+// nested filer operations that are part of a larger logical change, such as a
+// rename implemented via create+delete.
+func WithSuppressedMetadataEvents(ctx context.Context) context.Context {
+	return context.WithValue(ctx, suppressMetadataEventsKey{}, true)
+}
+
+func metadataEventsSuppressed(ctx context.Context) bool {
+	if ctx == nil {
+		return false
+	}
+	suppressed, _ := ctx.Value(suppressMetadataEventsKey{}).(bool)
+	return suppressed
+}

weed/filer/stream.go

@@ -141,12 +141,26 @@ func PrepareStreamContentWithThrottler(ctx context.Context, masterClient wdclien
 		var urlStrings []string
 		var err error
 		for _, backoff := range getLookupFileIdBackoffSchedule {
+			if err := ctx.Err(); err != nil {
+				return nil, err
+			}
 			urlStrings, err = masterClient.GetLookupFileIdFunction()(ctx, chunkView.FileId)
 			if err == nil && len(urlStrings) > 0 {
 				break
 			}
+			if err := ctx.Err(); err != nil {
+				return nil, err
+			}
 			glog.V(4).InfofCtx(ctx, "waiting for chunk: %s", chunkView.FileId)
-			time.Sleep(backoff)
+			timer := time.NewTimer(backoff)
+			select {
+			case <-ctx.Done():
+				if !timer.Stop() {
+					<-timer.C
+				}
+				return nil, ctx.Err()
+			case <-timer.C:
+			}
 		}
 		if err != nil {
 			glog.V(1).InfofCtx(ctx, "operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
@@ -179,6 +193,10 @@ func PrepareStreamContentWithThrottler(ctx context.Context, masterClient wdclien
 			jwt := jwtFunc(chunkView.FileId)
 			written, err := retriedStreamFetchChunkData(ctx, writer, urlStrings, jwt, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize))
 
+			if err != nil && ctx.Err() != nil {
+				return ctx.Err()
+			}
+
 			// If read failed, try to invalidate cache and re-lookup
 			if err != nil && written == 0 {
 				if invalidator, ok := masterClient.(CacheInvalidator); ok {

weed/filer/stream_failover_test.go

@@ -1,8 +1,11 @@
 package filer
 
 import (
+	"bytes"
 	"context"
+	"errors"
 	"testing"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
@@ -173,3 +176,106 @@ func TestRetryLogicSkipsSameUrls(t *testing.T) {
 		t.Error("Expected different URLs to not be equal")
 	}
 }
+
+func TestCanceledStreamSkipsCacheInvalidation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	fileId := "3,canceled"
+
+	mock := &mockMasterClient{
+		lookupFunc: func(ctx context.Context, fid string) ([]string, error) {
+			return []string{"http://server:8080"}, nil
+		},
+	}
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: fileId,
+			Offset: 0,
+			Size:   10,
+		},
+	}
+
+	streamFn, err := PrepareStreamContentWithThrottler(ctx, mock, noJwtFunc, chunks, 0, 10, 0)
+	if err != nil {
+		t.Fatalf("PrepareStreamContentWithThrottler failed: %v", err)
+	}
+
+	cancel()
+
+	err = streamFn(&bytes.Buffer{})
+	if err != context.Canceled {
+		t.Fatalf("expected context.Canceled, got %v", err)
+	}
+	if len(mock.invalidatedFileIds) != 0 {
+		t.Fatalf("expected no cache invalidation on cancellation, got %v", mock.invalidatedFileIds)
+	}
+}
+
+func TestPrepareStreamContentSkipsLookupWhenContextAlreadyCanceled(t *testing.T) {
+	oldSchedule := getLookupFileIdBackoffSchedule
+	getLookupFileIdBackoffSchedule = []time.Duration{time.Millisecond}
+	t.Cleanup(func() {
+		getLookupFileIdBackoffSchedule = oldSchedule
+	})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	lookupCalls := 0
+	mock := &mockMasterClient{
+		lookupFunc: func(ctx context.Context, fileId string) ([]string, error) {
+			lookupCalls++
+			return nil, errors.New("lookup should not run")
+		},
+	}
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: "3,precanceled",
+			Offset: 0,
+			Size:   10,
+		},
+	}
+
+	_, err := PrepareStreamContentWithThrottler(ctx, mock, noJwtFunc, chunks, 0, 10, 0)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected context.Canceled, got %v", err)
+	}
+	if lookupCalls != 0 {
+		t.Fatalf("expected no lookup calls after cancellation, got %d", lookupCalls)
+	}
+}
+
+func TestPrepareStreamContentStopsLookupRetriesAfterContextCancellation(t *testing.T) {
+	oldSchedule := getLookupFileIdBackoffSchedule
+	getLookupFileIdBackoffSchedule = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond}
+	t.Cleanup(func() {
+		getLookupFileIdBackoffSchedule = oldSchedule
+	})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	lookupCalls := 0
+	mock := &mockMasterClient{
+		lookupFunc: func(ctx context.Context, fileId string) ([]string, error) {
+			lookupCalls++
+			cancel()
+			return nil, context.Canceled
+		},
+	}
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: "3,cancel-during-lookup",
+			Offset: 0,
+			Size:   10,
+		},
+	}
+
+	_, err := PrepareStreamContentWithThrottler(ctx, mock, noJwtFunc, chunks, 0, 10, 0)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected context.Canceled, got %v", err)
+	}
+	if lookupCalls != 1 {
+		t.Fatalf("expected lookup retries to stop after cancellation, got %d calls", lookupCalls)
+	}
+}

weed/mount/filer_conf.go

@@ -46,19 +46,11 @@ func (wfs *WFS) subscribeFilerConfEvents() (*meta_cache.MetadataFollower, error)
 	}
 
 	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
-		message := resp.EventNotification
-		if message.NewEntry == nil {
+		if !isFilerConfUpdateEvent(resp, confDir, confName) {
 			return nil
 		}
 
-		dir := resp.Directory
-		name := resp.EventNotification.NewEntry.Name
-
-		if dir != confDir || name != confName {
-			return nil
-		}
-
-		content := message.NewEntry.Content
+		content := resp.EventNotification.NewEntry.Content
 		fc := filer.NewFilerConf()
 		if len(content) > 0 {
 			if err = fc.LoadFromBytes(content); err != nil {
@@ -76,6 +68,14 @@ func (wfs *WFS) subscribeFilerConfEvents() (*meta_cache.MetadataFollower, error)
 	}, nil
 }
 
+func isFilerConfUpdateEvent(resp *filer_pb.SubscribeMetadataResponse, confDir, confName string) bool {
+	if resp == nil || resp.EventNotification == nil || resp.EventNotification.NewEntry == nil {
+		return false
+	}
+	return filer_pb.MetadataEventTargetDirectory(resp) == confDir &&
+		resp.EventNotification.NewEntry.Name == confName
+}
+
 func (wfs *WFS) wormEnforcedForEntry(path util.FullPath, entry *filer_pb.Entry) (wormEnforced, wormEnabled bool) {
 	if entry == nil || wfs.FilerConf == nil {
 		return false, false

weed/mount/filer_conf_test.go

@@ -0,0 +1,23 @@
+package mount
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestIsFilerConfUpdateEventMatchesRenameTarget(t *testing.T) {
+	event := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/tmp",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: filer.FilerConfName},
+			NewEntry:      &filer_pb.Entry{Name: filer.FilerConfName},
+			NewParentPath: filer.DirectoryEtcSeaweedFS,
+		},
+	}
+
+	if !isFilerConfUpdateEvent(event, filer.DirectoryEtcSeaweedFS, filer.FilerConfName) {
+		t.Fatalf("expected rename target to match filer.conf watcher")
+	}
+}

weed/mount/meta_cache/meta_cache.go

@@ -27,18 +27,19 @@ type MetaCache struct {
 	localStore   filer.VirtualFilerStore
 	leveldbStore *leveldb.LevelDBStore // direct reference for batch operations
 	sync.RWMutex
-	uidGidMapper      *UidGidMapper
-	markCachedFn      func(fullpath util.FullPath)
-	isCachedFn        func(fullpath util.FullPath) bool
-	invalidateFunc    func(fullpath util.FullPath, entry *filer_pb.Entry)
-	onDirectoryUpdate func(dir util.FullPath)
-	visitGroup        singleflight.Group // deduplicates concurrent EnsureVisited calls for the same path
-	applyCh           chan metadataApplyRequest
-	applyDone         chan struct{}
-	applyStateMu      sync.Mutex
-	applyClosed       bool
-	buildingDirs      map[util.FullPath]*directoryBuildState
-	dedupRing         dedupRingBuffer
+	uidGidMapper         *UidGidMapper
+	markCachedFn         func(fullpath util.FullPath)
+	isCachedFn           func(fullpath util.FullPath) bool
+	invalidateFunc       func(fullpath util.FullPath, entry *filer_pb.Entry)
+	onDirectoryUpdate    func(dir util.FullPath)
+	visitGroup           singleflight.Group // deduplicates concurrent EnsureVisited calls for the same path
+	applyCh              chan metadataApplyRequest
+	applyDone            chan struct{}
+	applyStateMu         sync.Mutex
+	applyClosed          bool
+	buildingDirs         map[util.FullPath]*directoryBuildState
+	dedupRing            dedupRingBuffer
+	includeSystemEntries bool
 }
 
 var errMetaCacheClosed = errors.New("metadata cache is shut down")
@@ -84,17 +85,18 @@ type metadataApplyRequest struct {
 	done         chan error
 }
 
-func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPath,
+func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPath, includeSystemEntries bool,
 	markCachedFn func(path util.FullPath), isCachedFn func(path util.FullPath) bool, invalidateFunc func(util.FullPath, *filer_pb.Entry), onDirectoryUpdate func(dir util.FullPath)) *MetaCache {
 	leveldbStore, virtualStore := openMetaStore(dbFolder)
 	mc := &MetaCache{
-		root:              root,
-		localStore:        virtualStore,
-		leveldbStore:      leveldbStore,
-		markCachedFn:      markCachedFn,
-		isCachedFn:        isCachedFn,
-		uidGidMapper:      uidGidMapper,
-		onDirectoryUpdate: onDirectoryUpdate,
+		root:                 root,
+		localStore:           virtualStore,
+		leveldbStore:         leveldbStore,
+		markCachedFn:         markCachedFn,
+		isCachedFn:           isCachedFn,
+		uidGidMapper:         uidGidMapper,
+		onDirectoryUpdate:    onDirectoryUpdate,
+		includeSystemEntries: includeSystemEntries,
 		invalidateFunc: func(fullpath util.FullPath, entry *filer_pb.Entry) {
 			invalidateFunc(fullpath, entry)
 		},
@@ -182,6 +184,29 @@ func (mc *MetaCache) atomicUpdateEntryFromFilerLocked(ctx context.Context, oldPa
 	return nil
 }
 
+func (mc *MetaCache) shouldHideEntry(fullpath util.FullPath) bool {
+	if mc.includeSystemEntries {
+		return false
+	}
+	dir, name := fullpath.DirAndName()
+	return IsHiddenSystemEntry(dir, name)
+}
+
+func (mc *MetaCache) purgeEntryLocked(ctx context.Context, fullpath util.FullPath, isDirectory bool) error {
+	if fullpath == "" {
+		return nil
+	}
+	if err := mc.localStore.DeleteEntry(ctx, fullpath); err != nil {
+		return err
+	}
+	if isDirectory {
+		if err := mc.localStore.DeleteFolderChildren(ctx, fullpath); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (mc *MetaCache) ApplyMetadataResponse(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions) error {
 	if resp == nil || resp.EventNotification == nil {
 		return nil
@@ -520,7 +545,9 @@ func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *file
 	}
 
 	var oldPath util.FullPath
+	var newPath util.FullPath
 	var newEntry *filer.Entry
+	hideNewPath := false
 	if message.OldEntry != nil {
 		oldPath = util.NewFullPath(resp.Directory, message.OldEntry.Name)
 	}
@@ -530,11 +557,20 @@ func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *file
 		if message.NewParentPath != "" {
 			dir = message.NewParentPath
 		}
-		newEntry = filer.FromPbEntry(dir, message.NewEntry)
+		newPath = util.NewFullPath(dir, message.NewEntry.Name)
+		hideNewPath = mc.shouldHideEntry(newPath)
+		if !hideNewPath {
+			newEntry = filer.FromPbEntry(dir, message.NewEntry)
+		}
 	}
 
 	mc.Lock()
 	err := mc.atomicUpdateEntryFromFilerLocked(ctx, oldPath, newEntry, allowUncachedInsert)
+	if err == nil && hideNewPath {
+		if purgeErr := mc.purgeEntryLocked(ctx, newPath, message.NewEntry.IsDirectory); purgeErr != nil {
+			err = purgeErr
+		}
+	}
 	// When a directory is deleted or moved, remove its cached descendants
 	// so stale children cannot be served from the local cache.
 	if err == nil && oldPath != "" && message.OldEntry != nil && message.OldEntry.IsDirectory {

weed/mount/meta_cache/meta_cache_apply_test.go

@@ -2,6 +2,7 @@ package meta_cache
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"sync"
 	"testing"
@@ -296,6 +297,114 @@ func TestApplyMetadataResponseDeduplicatesRepeatedFilerEvent(t *testing.T) {
 	}
 }
 
+func TestApplyMetadataResponseSkipsHiddenSystemEntryWhenDisabled(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/": true,
+	})
+	defer mc.Shutdown()
+
+	createResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name: "topics",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    1,
+					FileMode: uint32(os.ModeDir | 0o755),
+				},
+				IsDirectory: true,
+			},
+		},
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), createResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply create: %v", err)
+	}
+
+	entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics"))
+	if err != filer_pb.ErrNotFound {
+		t.Fatalf("find hidden entry error = %v, want %v", err, filer_pb.ErrNotFound)
+	}
+	if entry != nil {
+		t.Fatalf("hidden entry still cached: %+v", entry)
+	}
+}
+
+func TestApplyMetadataResponsePurgesHiddenDestinationPath(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/src": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/topics",
+		Attr: filer.Attr{
+			Crtime: time.Unix(1, 0),
+			Mtime:  time.Unix(1, 0),
+			Mode:   os.ModeDir | 0o755,
+		},
+	}); err != nil {
+		t.Fatalf("insert stale hidden dir: %v", err)
+	}
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/topics/leaked.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0o644,
+			FileSize: 7,
+		},
+	}); err != nil {
+		t.Fatalf("insert leaked hidden child: %v", err)
+	}
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/src/visible",
+		Attr: filer.Attr{
+			Crtime: time.Unix(1, 0),
+			Mtime:  time.Unix(1, 0),
+			Mode:   os.ModeDir | 0o755,
+		},
+	}); err != nil {
+		t.Fatalf("insert source dir: %v", err)
+	}
+
+	renameResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/src",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name:        "visible",
+				IsDirectory: true,
+			},
+			NewEntry: &filer_pb.Entry{
+				Name: "topics",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   2,
+					Mtime:    2,
+					FileMode: uint32(os.ModeDir | 0o755),
+				},
+				IsDirectory: true,
+			},
+			NewParentPath: "/",
+		},
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), renameResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply rename: %v", err)
+	}
+
+	if entry, err := mc.FindEntry(context.Background(), util.FullPath("/src/visible")); err != filer_pb.ErrNotFound || entry != nil {
+		t.Fatalf("source dir after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
+	}
+	if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics")); err != filer_pb.ErrNotFound || entry != nil {
+		t.Fatalf("hidden destination after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
+	}
+	if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics/leaked.txt")); err != filer_pb.ErrNotFound || entry != nil {
+		t.Fatalf("hidden child after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
+	}
+}
+
 func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache, map[util.FullPath]bool, *recordedPaths, *recordedPaths) {
 	t.Helper()
 
@@ -312,6 +421,7 @@ func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache,
 		filepath.Join(t.TempDir(), "meta"),
 		mapper,
 		util.FullPath("/"),
+		false,
 		func(path util.FullPath) {
 			cachedMu.Lock()
 			defer cachedMu.Unlock()

weed/mount/meta_cache/meta_cache_init.go

@@ -107,7 +107,7 @@ func doEnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerCl
 			var err error
 			snapshotTsNs, err = filer_pb.ReadDirAllEntriesWithSnapshot(ctx, client, path, "", func(pbEntry *filer_pb.Entry, isLast bool) error {
 				entry := filer.FromPbEntry(string(path), pbEntry)
-				if IsHiddenSystemEntry(string(path), entry.Name()) {
+				if !mc.includeSystemEntries && IsHiddenSystemEntry(string(path), entry.Name()) {
 					return nil
 				}
 

weed/mount/weedfs.go

@@ -65,6 +65,7 @@ type Option struct {
 	VolumeServerAccess string // how to access volume servers
 	Cipher             bool   // whether encrypt data on volume server
 	UidGidMapper       *meta_cache.UidGidMapper
+	IncludeSystemEntries bool
 
 	// Periodic metadata flush interval in seconds (0 to disable)
 	// This protects chunks from being purged by volume.fsck for long-running writes
@@ -203,6 +204,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 
 	wfs.metaCache = meta_cache.NewMetaCache(path.Join(option.getUniqueCacheDirForRead(), "meta"), option.UidGidMapper,
 		util.FullPath(option.FilerMountRootPath),
+		option.IncludeSystemEntries,
 		func(path util.FullPath) {
 			wfs.inodeToPath.MarkChildrenCached(path)
 		}, func(path util.FullPath) bool {

weed/mount/weedfs_dir_read.go

@@ -302,7 +302,7 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out *fuse.DirEntryList,
 	if input.Offset >= dh.entryStreamOffset {
 		if len(dh.entryStream) == 0 && input.Offset > dh.entryStreamOffset {
 			skipCount := uint32(input.Offset-dh.entryStreamOffset) + batchSize
-			entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs)
+			entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
 			if err != nil {
 				glog.Errorf("list filer directory: %v", err)
 				return fuse.EIO
@@ -331,7 +331,7 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out *fuse.DirEntryList,
 			}
 		}
 
-		entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs)
+		entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
 		if err != nil {
 			glog.Errorf("list filer directory: %v", err)
 			return fuse.EIO
@@ -360,13 +360,13 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out *fuse.DirEntryList,
 	return fuse.OK
 }
 
-func loadDirectoryEntriesDirect(ctx context.Context, client filer_pb.FilerClient, uidGidMapper *meta_cache.UidGidMapper, dirPath util.FullPath, startFileName string, includeStart bool, limit uint32, snapshotTsNs int64) ([]*filer.Entry, int64, error) {
+func loadDirectoryEntriesDirect(ctx context.Context, client filer_pb.FilerClient, uidGidMapper *meta_cache.UidGidMapper, dirPath util.FullPath, startFileName string, includeStart bool, limit uint32, snapshotTsNs int64, includeSystemEntries bool) ([]*filer.Entry, int64, error) {
 	entries := make([]*filer.Entry, 0, limit)
 	var actualSnapshotTsNs int64
 	err := client.WithFilerClient(false, func(sc filer_pb.SeaweedFilerClient) error {
 		var innerErr error
 		actualSnapshotTsNs, innerErr = filer_pb.DoSeaweedListWithSnapshot(ctx, sc, dirPath, "", func(entry *filer_pb.Entry, isLast bool) error {
-			if meta_cache.IsHiddenSystemEntry(string(dirPath), entry.Name) {
+			if !includeSystemEntries && meta_cache.IsHiddenSystemEntry(string(dirPath), entry.Name) {
 				return nil
 			}
 			if uidGidMapper != nil && entry.Attributes != nil {

weed/mount/weedfs_dir_read_test.go

@@ -84,7 +84,7 @@ func TestLoadDirectoryEntriesDirectFiltersHiddenEntriesAndMapsIds(t *testing.T)
 		},
 	}
 
-	entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, mapper, util.FullPath("/"), "", false, 10, 0)
+	entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, mapper, util.FullPath("/"), "", false, 10, 0, false)
 	if err != nil {
 		t.Fatalf("loadDirectoryEntriesDirect: %v", err)
 	}
@@ -98,3 +98,25 @@ func TestLoadDirectoryEntriesDirectFiltersHiddenEntriesAndMapsIds(t *testing.T)
 		t.Fatalf("mapped uid/gid = %d/%d, want 10/20", entries[0].Attr.Uid, entries[0].Attr.Gid)
 	}
 }
+
+func TestLoadDirectoryEntriesDirectShowsSystemEntriesWhenEnabled(t *testing.T) {
+	client := &directoryFilerAccessor{
+		client: &directoryListClient{
+			responses: []*filer_pb.ListEntriesResponse{
+				{Entry: &filer_pb.Entry{Name: "topics"}},
+				{Entry: &filer_pb.Entry{Name: "visible"}},
+			},
+		},
+	}
+
+	entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, nil, util.FullPath("/"), "", false, 10, 0, true)
+	if err != nil {
+		t.Fatalf("loadDirectoryEntriesDirect: %v", err)
+	}
+	if got := len(entries); got != 2 {
+		t.Fatalf("entry count = %d, want 2", got)
+	}
+	if entries[0].Name() != "topics" || entries[1].Name() != "visible" {
+		t.Fatalf("entry names = %q, %q, want topics, visible", entries[0].Name(), entries[1].Name())
+	}
+}

weed/mount/weedfs_file_copy_range_test.go

@@ -338,6 +338,7 @@ func newCopyRangeTestWFSWithMetaCache(t *testing.T) *WFS {
 		filepath.Join(t.TempDir(), "meta"),
 		uidGidMapper,
 		root,
+		false,
 		func(path util.FullPath) {
 			wfs.inodeToPath.MarkChildrenCached(path)
 		},

weed/mount/weedfs_file_mkrm_test.go

@@ -120,6 +120,7 @@ func newCreateTestWFS(t *testing.T) (*WFS, *createEntryTestServer) {
 		filepath.Join(t.TempDir(), "meta"),
 		uidGidMapper,
 		root,
+		false,
 		func(path util.FullPath) {
 			wfs.inodeToPath.MarkChildrenCached(path)
 		},

weed/mount/weedfs_rename_test.go

@@ -23,6 +23,7 @@ func TestHandleRenameResponseLeavesUncachedTargetOutOfCache(t *testing.T) {
 		filepath.Join(t.TempDir(), "meta"),
 		uidGidMapper,
 		root,
+		false,
 		func(path util.FullPath) {
 			inodeToPath.MarkChildrenCached(path)
 		},

weed/notification/kafka/kafka_queue.go

@@ -1,6 +1,8 @@
 package kafka
 
 import (
+	"fmt"
+
 	"github.com/Shopify/sarama"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/notification"
@@ -27,15 +29,29 @@ func (k *KafkaQueue) Initialize(configuration util.Configuration, prefix string)
 	return k.initialize(
 		configuration.GetStringSlice(prefix+"hosts"),
 		configuration.GetString(prefix+"topic"),
+		SASLTLSConfig{
+			SASLEnabled:           configuration.GetBool(prefix + "sasl_enabled"),
+			SASLMechanism:         configuration.GetString(prefix + "sasl_mechanism"),
+			SASLUsername:          configuration.GetString(prefix + "sasl_username"),
+			SASLPassword:          configuration.GetString(prefix + "sasl_password"),
+			TLSEnabled:            configuration.GetBool(prefix + "tls_enabled"),
+			TLSCACert:             configuration.GetString(prefix + "tls_ca_cert"),
+			TLSClientCert:         configuration.GetString(prefix + "tls_client_cert"),
+			TLSClientKey:          configuration.GetString(prefix + "tls_client_key"),
+			TLSInsecureSkipVerify: configuration.GetBool(prefix + "tls_insecure_skip_verify"),
+		},
 	)
 }
 
-func (k *KafkaQueue) initialize(hosts []string, topic string) (err error) {
+func (k *KafkaQueue) initialize(hosts []string, topic string, saslTLS SASLTLSConfig) (err error) {
 	config := sarama.NewConfig()
 	config.Producer.RequiredAcks = sarama.WaitForLocal
 	config.Producer.Partitioner = sarama.NewHashPartitioner
 	config.Producer.Return.Successes = true
 	config.Producer.Return.Errors = true
+	if err = ConfigureSASLTLS(config, saslTLS); err != nil {
+		return fmt.Errorf("kafka producer security configuration: %w", err)
+	}
 	k.producer, err = sarama.NewAsyncProducer(hosts, config)
 	if err != nil {
 		return err

weed/notification/kafka/kafka_sasl_tls.go

@@ -0,0 +1,112 @@
+package kafka
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/Shopify/sarama"
+	"github.com/xdg-go/scram"
+)
+
+// SASLTLSConfig holds SASL and TLS configuration for Kafka connections.
+type SASLTLSConfig struct {
+	SASLEnabled   bool
+	SASLMechanism string
+	SASLUsername  string
+	SASLPassword  string
+
+	TLSEnabled            bool
+	TLSCACert             string
+	TLSClientCert         string
+	TLSClientKey          string
+	TLSInsecureSkipVerify bool
+}
+
+// ConfigureSASLTLS applies SASL and TLS settings to a sarama config.
+func ConfigureSASLTLS(config *sarama.Config, st SASLTLSConfig) error {
+	if st.SASLEnabled {
+		config.Net.SASL.Enable = true
+		config.Net.SASL.User = st.SASLUsername
+		config.Net.SASL.Password = st.SASLPassword
+
+		mechanism := strings.ToUpper(st.SASLMechanism)
+		switch mechanism {
+		case "PLAIN", "":
+			config.Net.SASL.Mechanism = sarama.SASLTypePlaintext
+		case "SCRAM-SHA-256":
+			config.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA256
+			config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
+				return &scramClient{HashGeneratorFcn: scram.SHA256}
+			}
+		case "SCRAM-SHA-512":
+			config.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA512
+			config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
+				return &scramClient{HashGeneratorFcn: scram.SHA512}
+			}
+		default:
+			return fmt.Errorf("unsupported SASL mechanism: %s", mechanism)
+		}
+	}
+
+	if st.TLSEnabled {
+		if (st.TLSClientCert == "") != (st.TLSClientKey == "") {
+			return fmt.Errorf("both tls_client_cert and tls_client_key must be provided for mTLS, or neither")
+		}
+
+		tlsConfig := &tls.Config{
+			MinVersion:         tls.VersionTLS12,
+			InsecureSkipVerify: st.TLSInsecureSkipVerify,
+		}
+
+		if st.TLSCACert != "" {
+			caCert, err := os.ReadFile(st.TLSCACert)
+			if err != nil {
+				return fmt.Errorf("failed to read CA certificate: %w", err)
+			}
+			caCertPool := x509.NewCertPool()
+			if !caCertPool.AppendCertsFromPEM(caCert) {
+				return fmt.Errorf("failed to parse CA certificate")
+			}
+			tlsConfig.RootCAs = caCertPool
+		}
+
+		if st.TLSClientCert != "" && st.TLSClientKey != "" {
+			cert, err := tls.LoadX509KeyPair(st.TLSClientCert, st.TLSClientKey)
+			if err != nil {
+				return fmt.Errorf("failed to load client certificate/key: %w", err)
+			}
+			tlsConfig.Certificates = []tls.Certificate{cert}
+		}
+
+		config.Net.TLS.Enable = true
+		config.Net.TLS.Config = tlsConfig
+	}
+
+	return nil
+}
+
+// scramClient implements the sarama.SCRAMClient interface.
+type scramClient struct {
+	*scram.ClientConversation
+	scram.HashGeneratorFcn
+}
+
+func (c *scramClient) Begin(userName, password, authzID string) (err error) {
+	client, err := c.HashGeneratorFcn.NewClient(userName, password, authzID)
+	if err != nil {
+		return err
+	}
+	c.ClientConversation = client.NewConversation()
+	return nil
+}
+
+func (c *scramClient) Step(challenge string) (string, error) {
+	return c.ClientConversation.Step(challenge)
+}
+
+func (c *scramClient) Done() bool {
+	return c.ClientConversation.Done()
+}

weed/notification/kafka/kafka_sasl_tls_test.go

@@ -0,0 +1,65 @@
+package kafka
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/Shopify/sarama"
+)
+
+func TestConfigureSASLTLSRejectsPartialMTLSConfig(t *testing.T) {
+	tests := []struct {
+		name string
+		cfg  SASLTLSConfig
+	}{
+		{
+			name: "missing key",
+			cfg: SASLTLSConfig{
+				TLSEnabled:    true,
+				TLSClientCert: "/tmp/client.crt",
+			},
+		},
+		{
+			name: "missing cert",
+			cfg: SASLTLSConfig{
+				TLSEnabled:   true,
+				TLSClientKey: "/tmp/client.key",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := ConfigureSASLTLS(sarama.NewConfig(), tt.cfg)
+			if err == nil {
+				t.Fatal("expected error")
+			}
+			if !strings.Contains(err.Error(), "both tls_client_cert and tls_client_key must be provided") {
+				t.Fatalf("unexpected error: %v", err)
+			}
+		})
+	}
+}
+
+func TestConfigureSASLTLSConfiguresSCRAMSHA256(t *testing.T) {
+	config := sarama.NewConfig()
+	err := ConfigureSASLTLS(config, SASLTLSConfig{
+		SASLEnabled:   true,
+		SASLMechanism: "SCRAM-SHA-256",
+		SASLUsername:  "alice",
+		SASLPassword:  "secret",
+	})
+	if err != nil {
+		t.Fatalf("ConfigureSASLTLS returned error: %v", err)
+	}
+
+	if !config.Net.SASL.Enable {
+		t.Fatal("expected SASL to be enabled")
+	}
+	if config.Net.SASL.Mechanism != sarama.SASLTypeSCRAMSHA256 {
+		t.Fatalf("unexpected mechanism: %v", config.Net.SASL.Mechanism)
+	}
+	if config.Net.SASL.SCRAMClientGeneratorFunc == nil {
+		t.Fatal("expected SCRAM client generator")
+	}
+}

weed/notification/webhook/filter.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 type filter struct {
@@ -40,25 +41,37 @@ func newFilter(cfg *config) *filter {
 }
 
 func (f *filter) shouldPublish(key string, notification *filer_pb.EventNotification) bool {
-	if !f.matchesPath(key) {
+	if !f.matchesPath(key, notification) {
 		return false
 	}
 
-	eventType := detectEventType(notification)
+	eventType := detectEventType(key, notification)
 
 	return f.eventTypes[eventType]
 }
 
-func (f *filter) matchesPath(key string) bool {
+func (f *filter) matchesPath(key string, notification *filer_pb.EventNotification) bool {
 	if len(f.pathPrefixes) == 0 {
 		return true
 	}
 
+	if f.matchesAnyPathPrefix(key) {
+		return true
+	}
+
+	if notification != nil && notification.NewEntry != nil && notification.NewParentPath != "" {
+		newKey := string(util.FullPath(notification.NewParentPath).Child(notification.NewEntry.Name))
+		return f.matchesAnyPathPrefix(newKey)
+	}
+
+	return false
+}
+
+func (f *filter) matchesAnyPathPrefix(key string) bool {
 	for _, prefix := range f.pathPrefixes {
 		if strings.HasPrefix(key, prefix) {
 			return true
 		}
 	}
-
 	return false
 }

weed/notification/webhook/filter_test.go

@@ -9,6 +9,7 @@ import (
 func TestFilterEventTypes(t *testing.T) {
 	tests := []struct {
 		name          string
+		key           string
 		eventTypes    []string
 		notification  *filer_pb.EventNotification
 		expectedType  eventType
@@ -16,6 +17,7 @@ func TestFilterEventTypes(t *testing.T) {
 	}{
 		{
 			name:       "create event - allowed",
+			key:        "/test/test.txt",
 			eventTypes: []string{"create", "delete"},
 			notification: &filer_pb.EventNotification{
 				NewEntry: &filer_pb.Entry{Name: "test.txt"},
@@ -25,6 +27,7 @@ func TestFilterEventTypes(t *testing.T) {
 		},
 		{
 			name:       "create event - not allowed",
+			key:        "/test/test.txt",
 			eventTypes: []string{"delete", "update"},
 			notification: &filer_pb.EventNotification{
 				NewEntry: &filer_pb.Entry{Name: "test.txt"},
@@ -34,6 +37,7 @@ func TestFilterEventTypes(t *testing.T) {
 		},
 		{
 			name:       "delete event - allowed",
+			key:        "/test/test.txt",
 			eventTypes: []string{"create", "delete"},
 			notification: &filer_pb.EventNotification{
 				OldEntry: &filer_pb.Entry{Name: "test.txt"},
@@ -43,16 +47,19 @@ func TestFilterEventTypes(t *testing.T) {
 		},
 		{
 			name:       "update event - allowed",
+			key:        "/test/test.txt",
 			eventTypes: []string{"update"},
 			notification: &filer_pb.EventNotification{
-				OldEntry: &filer_pb.Entry{Name: "test.txt"},
-				NewEntry: &filer_pb.Entry{Name: "test.txt"},
+				OldEntry:      &filer_pb.Entry{Name: "test.txt"},
+				NewEntry:      &filer_pb.Entry{Name: "test.txt"},
+				NewParentPath: "/test",
 			},
 			expectedType:  eventTypeUpdate,
 			shouldPublish: true,
 		},
 		{
 			name:       "rename event - allowed",
+			key:        "/old/path/old.txt",
 			eventTypes: []string{"rename"},
 			notification: &filer_pb.EventNotification{
 				OldEntry:      &filer_pb.Entry{Name: "old.txt"},
@@ -62,8 +69,21 @@ func TestFilterEventTypes(t *testing.T) {
 			expectedType:  eventTypeRename,
 			shouldPublish: true,
 		},
+		{
+			name:       "rename event same name different parent - allowed",
+			key:        "/old/path/file.txt",
+			eventTypes: []string{"rename"},
+			notification: &filer_pb.EventNotification{
+				OldEntry:      &filer_pb.Entry{Name: "file.txt"},
+				NewEntry:      &filer_pb.Entry{Name: "file.txt"},
+				NewParentPath: "/new/path",
+			},
+			expectedType:  eventTypeRename,
+			shouldPublish: true,
+		},
 		{
 			name:       "rename event - not allowed",
+			key:        "/old/path/old.txt",
 			eventTypes: []string{"create", "delete", "update"},
 			notification: &filer_pb.EventNotification{
 				OldEntry:      &filer_pb.Entry{Name: "old.txt"},
@@ -75,6 +95,7 @@ func TestFilterEventTypes(t *testing.T) {
 		},
 		{
 			name:       "all events allowed when empty",
+			key:        "/test/test.txt",
 			eventTypes: []string{},
 			notification: &filer_pb.EventNotification{
 				NewEntry: &filer_pb.Entry{Name: "test.txt"},
@@ -89,12 +110,12 @@ func TestFilterEventTypes(t *testing.T) {
 			cfg := &config{eventTypes: tt.eventTypes}
 			f := newFilter(cfg)
 
-			eventType := detectEventType(tt.notification)
+			eventType := detectEventType(tt.key, tt.notification)
 			if eventType != tt.expectedType {
 				t.Errorf("detectEventType() = %v, want %v", eventType, tt.expectedType)
 			}
 
-			shouldPublish := f.shouldPublish("/test/path", tt.notification)
+			shouldPublish := f.shouldPublish(tt.key, tt.notification)
 			if shouldPublish != tt.shouldPublish {
 				t.Errorf("shouldPublish() = %v, want %v", shouldPublish, tt.shouldPublish)
 			}
@@ -106,60 +127,82 @@ func TestFilterPathPrefixes(t *testing.T) {
 	tests := []struct {
 		name          string
 		pathPrefixes  []string
+		eventTypes    []string
 		key           string
+		notification  *filer_pb.EventNotification
 		shouldPublish bool
 	}{
 		{
 			name:          "matches single prefix",
 			pathPrefixes:  []string{"/data/"},
+			eventTypes:    []string{"create"},
 			key:           "/data/file.txt",
+			notification:  &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
 			shouldPublish: true,
 		},
 		{
 			name:          "matches one of multiple prefixes",
 			pathPrefixes:  []string{"/data/", "/logs/", "/tmp/"},
+			eventTypes:    []string{"create"},
 			key:           "/logs/app.log",
+			notification:  &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "app.log"}},
 			shouldPublish: true,
 		},
 		{
 			name:          "no match",
 			pathPrefixes:  []string{"/data/", "/logs/"},
+			eventTypes:    []string{"create"},
 			key:           "/other/file.txt",
+			notification:  &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
 			shouldPublish: false,
 		},
 		{
 			name:          "empty prefixes allows all",
 			pathPrefixes:  []string{},
+			eventTypes:    []string{"create"},
 			key:           "/any/path/file.txt",
+			notification:  &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
 			shouldPublish: true,
 		},
 		{
 			name:          "exact prefix match",
 			pathPrefixes:  []string{"/data"},
+			eventTypes:    []string{"create"},
 			key:           "/data",
+			notification:  &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "data"}},
 			shouldPublish: true,
 		},
 		{
 			name:          "partial match not allowed",
 			pathPrefixes:  []string{"/data/"},
+			eventTypes:    []string{"create"},
 			key:           "/database/file.txt",
+			notification:  &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
 			shouldPublish: false,
 		},
+		{
+			name:         "rename matches destination prefix",
+			pathPrefixes: []string{"/watched/"},
+			eventTypes:   []string{"rename"},
+			key:          "/outside/old.txt",
+			notification: &filer_pb.EventNotification{
+				OldEntry:      &filer_pb.Entry{Name: "old.txt"},
+				NewEntry:      &filer_pb.Entry{Name: "new.txt"},
+				NewParentPath: "/watched",
+			},
+			shouldPublish: true,
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cfg := &config{
 				pathPrefixes: tt.pathPrefixes,
-				eventTypes:   []string{"create"},
+				eventTypes:   tt.eventTypes,
 			}
 			f := newFilter(cfg)
 
-			notification := &filer_pb.EventNotification{
-				NewEntry: &filer_pb.Entry{Name: "test.txt"},
-			}
-
-			shouldPublish := f.shouldPublish(tt.key, notification)
+			shouldPublish := f.shouldPublish(tt.key, tt.notification)
 			if shouldPublish != tt.shouldPublish {
 				t.Errorf("shouldPublish() = %v, want %v", shouldPublish, tt.shouldPublish)
 			}

weed/notification/webhook/types.go

@@ -59,7 +59,7 @@ func newWebhookMessage(key string, message proto.Message) *webhookMessage {
 		return nil
 	}
 
-	eventType := string(detectEventType(notification))
+	eventType := string(detectEventType(key, notification))
 
 	return &webhookMessage{
 		Key:          key,
@@ -157,10 +157,9 @@ func (c *config) validate() error {
 	return nil
 }
 
-func detectEventType(notification *filer_pb.EventNotification) eventType {
+func detectEventType(key string, notification *filer_pb.EventNotification) eventType {
 	hasOldEntry := notification.OldEntry != nil
 	hasNewEntry := notification.NewEntry != nil
-	hasNewParentPath := notification.NewParentPath != ""
 
 	if !hasOldEntry && hasNewEntry {
 		return eventTypeCreate
@@ -171,7 +170,12 @@ func detectEventType(notification *filer_pb.EventNotification) eventType {
 	}
 
 	if hasOldEntry && hasNewEntry {
-		if hasNewParentPath {
+		oldDir, _ := util.FullPath(key).DirAndName()
+		newDir := notification.NewParentPath
+		if newDir == "" {
+			newDir = oldDir
+		}
+		if oldDir != newDir || notification.OldEntry.Name != notification.NewEntry.Name {
 			return eventTypeRename
 		}
 

weed/operation/upload_content.go

@@ -97,6 +97,13 @@ var (
 	once        sync.Once
 )
 
+var uploadRetryableAssignErrList = []string{
+	"transport",
+	"is read only",
+	"failed to write to local disk",
+	"Volume Size ",
+}
+
 // HTTPClient interface for testing
 type HTTPClient interface {
 	Do(req *http.Request) (*http.Response, error)
@@ -128,6 +135,34 @@ func newUploader(httpClient HTTPClient) *Uploader {
 	}
 }
 
+func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, host string, auth security.EncodedJwt, err error), uploadOption *UploadOption, genFileUrlFn func(host, fileId string) string, data []byte) (fileId string, uploadResult *UploadResult, err error) {
+	doUploadFunc := func() error {
+		var host string
+		var auth security.EncodedJwt
+		fileId, host, auth, err = assignFn()
+		if err != nil {
+			return err
+		}
+
+		uploadOption.UploadUrl = genFileUrlFn(host, fileId)
+		uploadOption.Jwt = auth
+
+		uploadResult, err = uploader.retriedUploadData(context.Background(), data, uploadOption)
+		return err
+	}
+
+	if uploadOption.RetryForever {
+		util.RetryUntil("uploadWithRetryForever", doUploadFunc, func(err error) (shouldContinue bool) {
+			glog.V(0).Infof("upload content: %v", err)
+			return true
+		})
+	} else {
+		err = util.MultiRetry("uploadWithRetry", uploadRetryableAssignErrList, doUploadFunc)
+	}
+
+	return
+}
+
 // UploadWithRetry will retry both assigning volume request and uploading content
 // The option parameter does not need to specify UploadUrl and Jwt, which will come from assigning volume.
 func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assignRequest *filer_pb.AssignVolumeRequest, uploadOption *UploadOption, genFileUrlFn func(host, fileId string) string, reader io.Reader) (fileId string, uploadResult *UploadResult, err error, data []byte) {
@@ -144,11 +179,7 @@ func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assi
 		glog.V(4).Infof("upload read %d bytes from %s", len(data), uploadOption.SourceUrl)
 	}
 
-	doUploadFunc := func() error {
-
-		var host string
-		var auth security.EncodedJwt
-
+	fileId, uploadResult, err = uploader.uploadWithRetryData(func() (fileId string, host string, auth security.EncodedJwt, err error) {
 		// grpc assign volume
 		if grpcAssignErr := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 			resp, assignErr := client.AssignVolume(context.Background(), assignRequest)
@@ -166,26 +197,10 @@ func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assi
 
 			return nil
 		}); grpcAssignErr != nil {
-			return fmt.Errorf("filerGrpcAddress assign volume: %w", grpcAssignErr)
+			err = fmt.Errorf("filerGrpcAddress assign volume: %w", grpcAssignErr)
 		}
-
-		uploadOption.UploadUrl = genFileUrlFn(host, fileId)
-		uploadOption.Jwt = auth
-
-		var uploadErr error
-		uploadResult, uploadErr = uploader.retriedUploadData(context.Background(), data, uploadOption)
-		return uploadErr
-	}
-	if uploadOption.RetryForever {
-		util.RetryUntil("uploadWithRetryForever", doUploadFunc, func(err error) (shouldContinue bool) {
-			glog.V(0).Infof("upload content: %v", err)
-			return true
-		})
-	} else {
-		uploadErrList := []string{"transport", "is read only"}
-		err = util.MultiRetry("uploadWithRetry", uploadErrList, doUploadFunc)
-	}
-
+		return
+	}, uploadOption, genFileUrlFn, data)
 	return
 }
 

weed/operation/upload_content_test.go

@@ -0,0 +1,124 @@
+package operation
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/security"
+)
+
+type scriptedHTTPResponse struct {
+	status int
+	body   string
+}
+
+type scriptedHTTPClient struct {
+	mu        sync.Mutex
+	responses map[string][]scriptedHTTPResponse
+	calls     []string
+}
+
+func testIsUploadRetryableAssignError(err error) bool {
+	if err == nil {
+		return false
+	}
+	for _, retryable := range uploadRetryableAssignErrList {
+		if strings.Contains(err.Error(), retryable) {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *scriptedHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	url := req.URL.String()
+	c.calls = append(c.calls, url)
+
+	plans := c.responses[url]
+	if len(plans) == 0 {
+		return nil, fmt.Errorf("unexpected request to %s", url)
+	}
+	plan := plans[0]
+	c.responses[url] = plans[1:]
+
+	return &http.Response{
+		StatusCode: plan.status,
+		Header:     make(http.Header),
+		Body:       io.NopCloser(strings.NewReader(plan.body)),
+	}, nil
+}
+
+func TestIsUploadRetryableAssignError(t *testing.T) {
+	testCases := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{name: "nil", err: nil, want: false},
+		{name: "transport", err: fmt.Errorf("transport is closing"), want: true},
+		{name: "read only", err: fmt.Errorf("volume 1 is read only"), want: true},
+		{name: "volume full", err: fmt.Errorf("failed to write to local disk: append to volume 1 size 0 actualSize 0: Volume Size 33555976 Exceeded 33554432"), want: true},
+		{name: "other permanent", err: fmt.Errorf("mismatching cookie"), want: false},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := testIsUploadRetryableAssignError(tc.err); got != tc.want {
+				t.Fatalf("testIsUploadRetryableAssignError(%v) = %v, want %v", tc.err, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestUploadWithRetryDataReassignsOnVolumeSizeExceeded(t *testing.T) {
+	httpClient := &scriptedHTTPClient{
+		responses: map[string][]scriptedHTTPResponse{
+			"http://volume-a/1,first": {
+				{status: http.StatusInternalServerError, body: `{"error":"failed to write to local disk: append to volume 1 size 0 actualSize 0: Volume Size 33555976 Exceeded 33554432"}`},
+				{status: http.StatusInternalServerError, body: `{"error":"failed to write to local disk: append to volume 1 size 0 actualSize 0: Volume Size 33555976 Exceeded 33554432"}`},
+				{status: http.StatusInternalServerError, body: `{"error":"failed to write to local disk: append to volume 1 size 0 actualSize 0: Volume Size 33555976 Exceeded 33554432"}`},
+			},
+			"http://volume-b/2,second": {
+				{status: http.StatusCreated, body: `{"name":"test.bin","size":3}`},
+			},
+		},
+	}
+	uploader := newUploader(httpClient)
+
+	assignCalls := 0
+	fileID, uploadResult, err := uploader.uploadWithRetryData(func() (string, string, security.EncodedJwt, error) {
+		assignCalls++
+		if assignCalls == 1 {
+			return "1,first", "volume-a", "", nil
+		}
+		return "2,second", "volume-b", "", nil
+	}, &UploadOption{Filename: "test.bin"}, func(host, fileId string) string {
+		return "http://" + host + "/" + fileId
+	}, []byte("abc"))
+
+	if err != nil {
+		t.Fatalf("expected success after reassignment, got %v", err)
+	}
+	if fileID != "2,second" {
+		t.Fatalf("expected second file id, got %s", fileID)
+	}
+	if assignCalls != 2 {
+		t.Fatalf("expected 2 assign attempts, got %d", assignCalls)
+	}
+	if uploadResult == nil || uploadResult.Name != "test.bin" {
+		t.Fatalf("expected successful upload result, got %#v", uploadResult)
+	}
+	if len(httpClient.calls) != 4 {
+		t.Fatalf("expected 4 upload attempts (3 same-url retries + 1 reassigned upload), got %d", len(httpClient.calls))
+	}
+	if httpClient.calls[0] != "http://volume-a/1,first" || httpClient.calls[3] != "http://volume-b/2,second" {
+		t.Fatalf("unexpected upload call sequence: %#v", httpClient.calls)
+	}
+}

weed/pb/filer.proto

@@ -84,6 +84,8 @@ service SeaweedFiler {
     // distributed lock management internal use only
     rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
     }
+    rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
+    }
 }
 
 //////////////////////////////////////////////////
@@ -522,6 +524,7 @@ message LockResponse {
     string lock_owner = 2;
     string lock_host_moved_to = 3;
     string error = 4;
+    int64 generation = 5;
 }
 message UnlockRequest {
     string name = 1;
@@ -544,12 +547,26 @@ message Lock {
     string renew_token = 2;
     int64 expired_at_ns = 3;
     string owner = 4;
+    int64 generation = 5;
+    bool is_backup = 6;
+    int64 seq = 7;
 }
 message TransferLocksRequest {
     repeated Lock locks = 1;
 }
 message TransferLocksResponse {
 }
+message ReplicateLockRequest {
+    string name = 1;
+    string renew_token = 2;
+    int64 expired_at_ns = 3;
+    string owner = 4;
+    int64 generation = 5;
+    bool is_unlock = 6;
+    int64 seq = 7;
+}
+message ReplicateLockResponse {
+}
 
 //////////////////////////////////////////////////
 // StreamMutateEntry: ordered bidirectional streaming for all filer mutations.

weed/pb/filer_pb/filer.pb.go

@@ -3991,6 +3991,7 @@ type LockResponse struct {
 	LockOwner       string                 `protobuf:"bytes,2,opt,name=lock_owner,json=lockOwner,proto3" json:"lock_owner,omitempty"`
 	LockHostMovedTo string                 `protobuf:"bytes,3,opt,name=lock_host_moved_to,json=lockHostMovedTo,proto3" json:"lock_host_moved_to,omitempty"`
 	Error           string                 `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"`
+	Generation      int64                  `protobuf:"varint,5,opt,name=generation,proto3" json:"generation,omitempty"`
 	unknownFields   protoimpl.UnknownFields
 	sizeCache       protoimpl.SizeCache
 }
@@ -4053,6 +4054,13 @@ func (x *LockResponse) GetError() string {
 	return ""
 }
 
+func (x *LockResponse) GetGeneration() int64 {
+	if x != nil {
+		return x.Generation
+	}
+	return 0
+}
+
 type UnlockRequest struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	Name          string                 `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
@@ -4267,6 +4275,9 @@ type Lock struct {
 	RenewToken    string                 `protobuf:"bytes,2,opt,name=renew_token,json=renewToken,proto3" json:"renew_token,omitempty"`
 	ExpiredAtNs   int64                  `protobuf:"varint,3,opt,name=expired_at_ns,json=expiredAtNs,proto3" json:"expired_at_ns,omitempty"`
 	Owner         string                 `protobuf:"bytes,4,opt,name=owner,proto3" json:"owner,omitempty"`
+	Generation    int64                  `protobuf:"varint,5,opt,name=generation,proto3" json:"generation,omitempty"`
+	IsBackup      bool                   `protobuf:"varint,6,opt,name=is_backup,json=isBackup,proto3" json:"is_backup,omitempty"`
+	Seq           int64                  `protobuf:"varint,7,opt,name=seq,proto3" json:"seq,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
@@ -4329,6 +4340,27 @@ func (x *Lock) GetOwner() string {
 	return ""
 }
 
+func (x *Lock) GetGeneration() int64 {
+	if x != nil {
+		return x.Generation
+	}
+	return 0
+}
+
+func (x *Lock) GetIsBackup() bool {
+	if x != nil {
+		return x.IsBackup
+	}
+	return false
+}
+
+func (x *Lock) GetSeq() int64 {
+	if x != nil {
+		return x.Seq
+	}
+	return 0
+}
+
 type TransferLocksRequest struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	Locks         []*Lock                `protobuf:"bytes,1,rep,name=locks,proto3" json:"locks,omitempty"`
@@ -4409,6 +4441,134 @@ func (*TransferLocksResponse) Descriptor() ([]byte, []int) {
 	return file_filer_proto_rawDescGZIP(), []int{66}
 }
 
+type ReplicateLockRequest struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Name          string                 `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	RenewToken    string                 `protobuf:"bytes,2,opt,name=renew_token,json=renewToken,proto3" json:"renew_token,omitempty"`
+	ExpiredAtNs   int64                  `protobuf:"varint,3,opt,name=expired_at_ns,json=expiredAtNs,proto3" json:"expired_at_ns,omitempty"`
+	Owner         string                 `protobuf:"bytes,4,opt,name=owner,proto3" json:"owner,omitempty"`
+	Generation    int64                  `protobuf:"varint,5,opt,name=generation,proto3" json:"generation,omitempty"`
+	IsUnlock      bool                   `protobuf:"varint,6,opt,name=is_unlock,json=isUnlock,proto3" json:"is_unlock,omitempty"`
+	Seq           int64                  `protobuf:"varint,7,opt,name=seq,proto3" json:"seq,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ReplicateLockRequest) Reset() {
+	*x = ReplicateLockRequest{}
+	mi := &file_filer_proto_msgTypes[67]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ReplicateLockRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ReplicateLockRequest) ProtoMessage() {}
+
+func (x *ReplicateLockRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_filer_proto_msgTypes[67]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ReplicateLockRequest.ProtoReflect.Descriptor instead.
+func (*ReplicateLockRequest) Descriptor() ([]byte, []int) {
+	return file_filer_proto_rawDescGZIP(), []int{67}
+}
+
+func (x *ReplicateLockRequest) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *ReplicateLockRequest) GetRenewToken() string {
+	if x != nil {
+		return x.RenewToken
+	}
+	return ""
+}
+
+func (x *ReplicateLockRequest) GetExpiredAtNs() int64 {
+	if x != nil {
+		return x.ExpiredAtNs
+	}
+	return 0
+}
+
+func (x *ReplicateLockRequest) GetOwner() string {
+	if x != nil {
+		return x.Owner
+	}
+	return ""
+}
+
+func (x *ReplicateLockRequest) GetGeneration() int64 {
+	if x != nil {
+		return x.Generation
+	}
+	return 0
+}
+
+func (x *ReplicateLockRequest) GetIsUnlock() bool {
+	if x != nil {
+		return x.IsUnlock
+	}
+	return false
+}
+
+func (x *ReplicateLockRequest) GetSeq() int64 {
+	if x != nil {
+		return x.Seq
+	}
+	return 0
+}
+
+type ReplicateLockResponse struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ReplicateLockResponse) Reset() {
+	*x = ReplicateLockResponse{}
+	mi := &file_filer_proto_msgTypes[68]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ReplicateLockResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ReplicateLockResponse) ProtoMessage() {}
+
+func (x *ReplicateLockResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_filer_proto_msgTypes[68]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ReplicateLockResponse.ProtoReflect.Descriptor instead.
+func (*ReplicateLockResponse) Descriptor() ([]byte, []int) {
+	return file_filer_proto_rawDescGZIP(), []int{68}
+}
+
 type StreamMutateEntryRequest struct {
 	state     protoimpl.MessageState `protogen:"open.v1"`
 	RequestId uint64                 `protobuf:"varint,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
@@ -4425,7 +4585,7 @@ type StreamMutateEntryRequest struct {
 
 func (x *StreamMutateEntryRequest) Reset() {
 	*x = StreamMutateEntryRequest{}
-	mi := &file_filer_proto_msgTypes[67]
+	mi := &file_filer_proto_msgTypes[69]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -4437,7 +4597,7 @@ func (x *StreamMutateEntryRequest) String() string {
 func (*StreamMutateEntryRequest) ProtoMessage() {}
 
 func (x *StreamMutateEntryRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_filer_proto_msgTypes[67]
+	mi := &file_filer_proto_msgTypes[69]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -4450,7 +4610,7 @@ func (x *StreamMutateEntryRequest) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use StreamMutateEntryRequest.ProtoReflect.Descriptor instead.
 func (*StreamMutateEntryRequest) Descriptor() ([]byte, []int) {
-	return file_filer_proto_rawDescGZIP(), []int{67}
+	return file_filer_proto_rawDescGZIP(), []int{69}
 }
 
 func (x *StreamMutateEntryRequest) GetRequestId() uint64 {
@@ -4550,7 +4710,7 @@ type StreamMutateEntryResponse struct {
 
 func (x *StreamMutateEntryResponse) Reset() {
 	*x = StreamMutateEntryResponse{}
-	mi := &file_filer_proto_msgTypes[68]
+	mi := &file_filer_proto_msgTypes[70]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -4562,7 +4722,7 @@ func (x *StreamMutateEntryResponse) String() string {
 func (*StreamMutateEntryResponse) ProtoMessage() {}
 
 func (x *StreamMutateEntryResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_filer_proto_msgTypes[68]
+	mi := &file_filer_proto_msgTypes[70]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -4575,7 +4735,7 @@ func (x *StreamMutateEntryResponse) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use StreamMutateEntryResponse.ProtoReflect.Descriptor instead.
 func (*StreamMutateEntryResponse) Descriptor() ([]byte, []int) {
-	return file_filer_proto_rawDescGZIP(), []int{68}
+	return file_filer_proto_rawDescGZIP(), []int{70}
 }
 
 func (x *StreamMutateEntryResponse) GetRequestId() uint64 {
@@ -4689,7 +4849,7 @@ type LocateBrokerResponse_Resource struct {
 
 func (x *LocateBrokerResponse_Resource) Reset() {
 	*x = LocateBrokerResponse_Resource{}
-	mi := &file_filer_proto_msgTypes[72]
+	mi := &file_filer_proto_msgTypes[74]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -4701,7 +4861,7 @@ func (x *LocateBrokerResponse_Resource) String() string {
 func (*LocateBrokerResponse_Resource) ProtoMessage() {}
 
 func (x *LocateBrokerResponse_Resource) ProtoReflect() protoreflect.Message {
-	mi := &file_filer_proto_msgTypes[72]
+	mi := &file_filer_proto_msgTypes[74]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -4755,7 +4915,7 @@ type FilerConf_PathConf struct {
 
 func (x *FilerConf_PathConf) Reset() {
 	*x = FilerConf_PathConf{}
-	mi := &file_filer_proto_msgTypes[73]
+	mi := &file_filer_proto_msgTypes[75]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -4767,7 +4927,7 @@ func (x *FilerConf_PathConf) String() string {
 func (*FilerConf_PathConf) ProtoMessage() {}
 
 func (x *FilerConf_PathConf) ProtoReflect() protoreflect.Message {
-	mi := &file_filer_proto_msgTypes[73]
+	mi := &file_filer_proto_msgTypes[75]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -5258,14 +5418,17 @@ const file_filer_proto_rawDesc = "" +
 	"\vrenew_token\x18\x03 \x01(\tR\n" +
 	"renewToken\x12\x19\n" +
 	"\bis_moved\x18\x04 \x01(\bR\aisMoved\x12\x14\n" +
-	"\x05owner\x18\x05 \x01(\tR\x05owner\"\x91\x01\n" +
+	"\x05owner\x18\x05 \x01(\tR\x05owner\"\xb1\x01\n" +
 	"\fLockResponse\x12\x1f\n" +
 	"\vrenew_token\x18\x01 \x01(\tR\n" +
 	"renewToken\x12\x1d\n" +
 	"\n" +
 	"lock_owner\x18\x02 \x01(\tR\tlockOwner\x12+\n" +
 	"\x12lock_host_moved_to\x18\x03 \x01(\tR\x0flockHostMovedTo\x12\x14\n" +
-	"\x05error\x18\x04 \x01(\tR\x05error\"_\n" +
+	"\x05error\x18\x04 \x01(\tR\x05error\x12\x1e\n" +
+	"\n" +
+	"generation\x18\x05 \x01(\x03R\n" +
+	"generation\"_\n" +
 	"\rUnlockRequest\x12\x12\n" +
 	"\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n" +
 	"\vrenew_token\x18\x02 \x01(\tR\n" +
@@ -5278,16 +5441,33 @@ const file_filer_proto_rawDesc = "" +
 	"\x04name\x18\x01 \x01(\tR\x04name\x12\x19\n" +
 	"\bis_moved\x18\x02 \x01(\bR\aisMoved\"-\n" +
 	"\x15FindLockOwnerResponse\x12\x14\n" +
-	"\x05owner\x18\x01 \x01(\tR\x05owner\"u\n" +
+	"\x05owner\x18\x01 \x01(\tR\x05owner\"\xc4\x01\n" +
 	"\x04Lock\x12\x12\n" +
 	"\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n" +
 	"\vrenew_token\x18\x02 \x01(\tR\n" +
 	"renewToken\x12\"\n" +
 	"\rexpired_at_ns\x18\x03 \x01(\x03R\vexpiredAtNs\x12\x14\n" +
-	"\x05owner\x18\x04 \x01(\tR\x05owner\"<\n" +
+	"\x05owner\x18\x04 \x01(\tR\x05owner\x12\x1e\n" +
+	"\n" +
+	"generation\x18\x05 \x01(\x03R\n" +
+	"generation\x12\x1b\n" +
+	"\tis_backup\x18\x06 \x01(\bR\bisBackup\x12\x10\n" +
+	"\x03seq\x18\a \x01(\x03R\x03seq\"<\n" +
 	"\x14TransferLocksRequest\x12$\n" +
 	"\x05locks\x18\x01 \x03(\v2\x0e.filer_pb.LockR\x05locks\"\x17\n" +
-	"\x15TransferLocksResponse\"\xe6\x02\n" +
+	"\x15TransferLocksResponse\"\xd4\x01\n" +
+	"\x14ReplicateLockRequest\x12\x12\n" +
+	"\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n" +
+	"\vrenew_token\x18\x02 \x01(\tR\n" +
+	"renewToken\x12\"\n" +
+	"\rexpired_at_ns\x18\x03 \x01(\x03R\vexpiredAtNs\x12\x14\n" +
+	"\x05owner\x18\x04 \x01(\tR\x05owner\x12\x1e\n" +
+	"\n" +
+	"generation\x18\x05 \x01(\x03R\n" +
+	"generation\x12\x1b\n" +
+	"\tis_unlock\x18\x06 \x01(\bR\bisUnlock\x12\x10\n" +
+	"\x03seq\x18\a \x01(\x03R\x03seq\"\x17\n" +
+	"\x15ReplicateLockResponse\"\xe6\x02\n" +
 	"\x18StreamMutateEntryRequest\x12\x1d\n" +
 	"\n" +
 	"request_id\x18\x01 \x01(\x04R\trequestId\x12E\n" +
@@ -5321,7 +5501,7 @@ const file_filer_proto_rawDesc = "" +
 	"\x0ePARENT_IS_FILE\x10\x02\x12\x19\n" +
 	"\x15EXISTING_IS_DIRECTORY\x10\x03\x12\x14\n" +
 	"\x10EXISTING_IS_FILE\x10\x04\x12\x18\n" +
-	"\x14ENTRY_ALREADY_EXISTS\x10\x052\xdb\x11\n" +
+	"\x14ENTRY_ALREADY_EXISTS\x10\x052\xaf\x12\n" +
 	"\fSeaweedFiler\x12g\n" +
 	"\x14LookupDirectoryEntry\x12%.filer_pb.LookupDirectoryEntryRequest\x1a&.filer_pb.LookupDirectoryEntryResponse\"\x00\x12N\n" +
 	"\vListEntries\x12\x1c.filer_pb.ListEntriesRequest\x1a\x1d.filer_pb.ListEntriesResponse\"\x000\x01\x12L\n" +
@@ -5349,7 +5529,8 @@ const file_filer_proto_rawDesc = "" +
 	"\x0fDistributedLock\x12\x15.filer_pb.LockRequest\x1a\x16.filer_pb.LockResponse\"\x00\x12H\n" +
 	"\x11DistributedUnlock\x12\x17.filer_pb.UnlockRequest\x1a\x18.filer_pb.UnlockResponse\"\x00\x12R\n" +
 	"\rFindLockOwner\x12\x1e.filer_pb.FindLockOwnerRequest\x1a\x1f.filer_pb.FindLockOwnerResponse\"\x00\x12R\n" +
-	"\rTransferLocks\x12\x1e.filer_pb.TransferLocksRequest\x1a\x1f.filer_pb.TransferLocksResponse\"\x00BO\n" +
+	"\rTransferLocks\x12\x1e.filer_pb.TransferLocksRequest\x1a\x1f.filer_pb.TransferLocksResponse\"\x00\x12R\n" +
+	"\rReplicateLock\x12\x1e.filer_pb.ReplicateLockRequest\x1a\x1f.filer_pb.ReplicateLockResponse\"\x00BO\n" +
 	"\x10seaweedfs.clientB\n" +
 	"FilerProtoZ/github.com/seaweedfs/seaweedfs/weed/pb/filer_pbb\x06proto3"
 
@@ -5366,7 +5547,7 @@ func file_filer_proto_rawDescGZIP() []byte {
 }
 
 var file_filer_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
-var file_filer_proto_msgTypes = make([]protoimpl.MessageInfo, 74)
+var file_filer_proto_msgTypes = make([]protoimpl.MessageInfo, 76)
 var file_filer_proto_goTypes = []any{
 	(SSEType)(0),                                    // 0: filer_pb.SSEType
 	(FilerError)(0),                                 // 1: filer_pb.FilerError
@@ -5437,20 +5618,22 @@ var file_filer_proto_goTypes = []any{
 	(*Lock)(nil),                                    // 66: filer_pb.Lock
 	(*TransferLocksRequest)(nil),                    // 67: filer_pb.TransferLocksRequest
 	(*TransferLocksResponse)(nil),                   // 68: filer_pb.TransferLocksResponse
-	(*StreamMutateEntryRequest)(nil),                // 69: filer_pb.StreamMutateEntryRequest
-	(*StreamMutateEntryResponse)(nil),               // 70: filer_pb.StreamMutateEntryResponse
-	nil,                                             // 71: filer_pb.Entry.ExtendedEntry
-	nil,                                             // 72: filer_pb.UpdateEntryRequest.ExpectedExtendedEntry
-	nil,                                             // 73: filer_pb.LookupVolumeResponse.LocationsMapEntry
-	(*LocateBrokerResponse_Resource)(nil),           // 74: filer_pb.LocateBrokerResponse.Resource
-	(*FilerConf_PathConf)(nil),                      // 75: filer_pb.FilerConf.PathConf
+	(*ReplicateLockRequest)(nil),                    // 69: filer_pb.ReplicateLockRequest
+	(*ReplicateLockResponse)(nil),                   // 70: filer_pb.ReplicateLockResponse
+	(*StreamMutateEntryRequest)(nil),                // 71: filer_pb.StreamMutateEntryRequest
+	(*StreamMutateEntryResponse)(nil),               // 72: filer_pb.StreamMutateEntryResponse
+	nil,                                             // 73: filer_pb.Entry.ExtendedEntry
+	nil,                                             // 74: filer_pb.UpdateEntryRequest.ExpectedExtendedEntry
+	nil,                                             // 75: filer_pb.LookupVolumeResponse.LocationsMapEntry
+	(*LocateBrokerResponse_Resource)(nil),           // 76: filer_pb.LocateBrokerResponse.Resource
+	(*FilerConf_PathConf)(nil),                      // 77: filer_pb.FilerConf.PathConf
 }
 var file_filer_proto_depIdxs = []int32{
 	7,  // 0: filer_pb.LookupDirectoryEntryResponse.entry:type_name -> filer_pb.Entry
 	7,  // 1: filer_pb.ListEntriesResponse.entry:type_name -> filer_pb.Entry
 	10, // 2: filer_pb.Entry.chunks:type_name -> filer_pb.FileChunk
 	13, // 3: filer_pb.Entry.attributes:type_name -> filer_pb.FuseAttributes
-	71, // 4: filer_pb.Entry.extended:type_name -> filer_pb.Entry.ExtendedEntry
+	73, // 4: filer_pb.Entry.extended:type_name -> filer_pb.Entry.ExtendedEntry
 	6,  // 5: filer_pb.Entry.remote_entry:type_name -> filer_pb.RemoteEntry
 	7,  // 6: filer_pb.FullEntry.entry:type_name -> filer_pb.Entry
 	7,  // 7: filer_pb.EventNotification.old_entry:type_name -> filer_pb.Entry
@@ -5463,22 +5646,22 @@ var file_filer_proto_depIdxs = []int32{
 	44, // 14: filer_pb.CreateEntryResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse
 	1,  // 15: filer_pb.CreateEntryResponse.error_code:type_name -> filer_pb.FilerError
 	7,  // 16: filer_pb.UpdateEntryRequest.entry:type_name -> filer_pb.Entry
-	72, // 17: filer_pb.UpdateEntryRequest.expected_extended:type_name -> filer_pb.UpdateEntryRequest.ExpectedExtendedEntry
+	74, // 17: filer_pb.UpdateEntryRequest.expected_extended:type_name -> filer_pb.UpdateEntryRequest.ExpectedExtendedEntry
 	44, // 18: filer_pb.UpdateEntryResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse
 	10, // 19: filer_pb.AppendToEntryRequest.chunks:type_name -> filer_pb.FileChunk
 	44, // 20: filer_pb.DeleteEntryResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse
 	9,  // 21: filer_pb.StreamRenameEntryResponse.event_notification:type_name -> filer_pb.EventNotification
 	30, // 22: filer_pb.AssignVolumeResponse.location:type_name -> filer_pb.Location
 	30, // 23: filer_pb.Locations.locations:type_name -> filer_pb.Location
-	73, // 24: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry
+	75, // 24: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry
 	32, // 25: filer_pb.CollectionListResponse.collections:type_name -> filer_pb.Collection
 	9,  // 26: filer_pb.SubscribeMetadataResponse.event_notification:type_name -> filer_pb.EventNotification
 	44, // 27: filer_pb.SubscribeMetadataResponse.events:type_name -> filer_pb.SubscribeMetadataResponse
 	45, // 28: filer_pb.SubscribeMetadataResponse.log_file_refs:type_name -> filer_pb.LogFileChunkRef
 	10, // 29: filer_pb.LogFileChunkRef.chunks:type_name -> filer_pb.FileChunk
 	7,  // 30: filer_pb.TraverseBfsMetadataResponse.entry:type_name -> filer_pb.Entry
-	74, // 31: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource
-	75, // 32: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf
+	76, // 31: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource
+	77, // 32: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf
 	7,  // 33: filer_pb.CacheRemoteObjectToLocalClusterResponse.entry:type_name -> filer_pb.Entry
 	44, // 34: filer_pb.CacheRemoteObjectToLocalClusterResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse
 	66, // 35: filer_pb.TransferLocksRequest.locks:type_name -> filer_pb.Lock
@@ -5499,7 +5682,7 @@ var file_filer_proto_depIdxs = []int32{
 	20, // 50: filer_pb.SeaweedFiler.DeleteEntry:input_type -> filer_pb.DeleteEntryRequest
 	22, // 51: filer_pb.SeaweedFiler.AtomicRenameEntry:input_type -> filer_pb.AtomicRenameEntryRequest
 	24, // 52: filer_pb.SeaweedFiler.StreamRenameEntry:input_type -> filer_pb.StreamRenameEntryRequest
-	69, // 53: filer_pb.SeaweedFiler.StreamMutateEntry:input_type -> filer_pb.StreamMutateEntryRequest
+	71, // 53: filer_pb.SeaweedFiler.StreamMutateEntry:input_type -> filer_pb.StreamMutateEntryRequest
 	26, // 54: filer_pb.SeaweedFiler.AssignVolume:input_type -> filer_pb.AssignVolumeRequest
 	28, // 55: filer_pb.SeaweedFiler.LookupVolume:input_type -> filer_pb.LookupVolumeRequest
 	33, // 56: filer_pb.SeaweedFiler.CollectionList:input_type -> filer_pb.CollectionListRequest
@@ -5517,34 +5700,36 @@ var file_filer_proto_depIdxs = []int32{
 	62, // 68: filer_pb.SeaweedFiler.DistributedUnlock:input_type -> filer_pb.UnlockRequest
 	64, // 69: filer_pb.SeaweedFiler.FindLockOwner:input_type -> filer_pb.FindLockOwnerRequest
 	67, // 70: filer_pb.SeaweedFiler.TransferLocks:input_type -> filer_pb.TransferLocksRequest
-	3,  // 71: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse
-	5,  // 72: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse
-	15, // 73: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse
-	17, // 74: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse
-	19, // 75: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse
-	21, // 76: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse
-	23, // 77: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse
-	25, // 78: filer_pb.SeaweedFiler.StreamRenameEntry:output_type -> filer_pb.StreamRenameEntryResponse
-	70, // 79: filer_pb.SeaweedFiler.StreamMutateEntry:output_type -> filer_pb.StreamMutateEntryResponse
-	27, // 80: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse
-	31, // 81: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse
-	34, // 82: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse
-	36, // 83: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse
-	38, // 84: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse
-	40, // 85: filer_pb.SeaweedFiler.Ping:output_type -> filer_pb.PingResponse
-	42, // 86: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse
-	47, // 87: filer_pb.SeaweedFiler.TraverseBfsMetadata:output_type -> filer_pb.TraverseBfsMetadataResponse
-	44, // 88: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse
-	44, // 89: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse
-	54, // 90: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse
-	56, // 91: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse
-	59, // 92: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:output_type -> filer_pb.CacheRemoteObjectToLocalClusterResponse
-	61, // 93: filer_pb.SeaweedFiler.DistributedLock:output_type -> filer_pb.LockResponse
-	63, // 94: filer_pb.SeaweedFiler.DistributedUnlock:output_type -> filer_pb.UnlockResponse
-	65, // 95: filer_pb.SeaweedFiler.FindLockOwner:output_type -> filer_pb.FindLockOwnerResponse
-	68, // 96: filer_pb.SeaweedFiler.TransferLocks:output_type -> filer_pb.TransferLocksResponse
-	71, // [71:97] is the sub-list for method output_type
-	45, // [45:71] is the sub-list for method input_type
+	69, // 71: filer_pb.SeaweedFiler.ReplicateLock:input_type -> filer_pb.ReplicateLockRequest
+	3,  // 72: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse
+	5,  // 73: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse
+	15, // 74: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse
+	17, // 75: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse
+	19, // 76: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse
+	21, // 77: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse
+	23, // 78: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse
+	25, // 79: filer_pb.SeaweedFiler.StreamRenameEntry:output_type -> filer_pb.StreamRenameEntryResponse
+	72, // 80: filer_pb.SeaweedFiler.StreamMutateEntry:output_type -> filer_pb.StreamMutateEntryResponse
+	27, // 81: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse
+	31, // 82: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse
+	34, // 83: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse
+	36, // 84: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse
+	38, // 85: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse
+	40, // 86: filer_pb.SeaweedFiler.Ping:output_type -> filer_pb.PingResponse
+	42, // 87: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse
+	47, // 88: filer_pb.SeaweedFiler.TraverseBfsMetadata:output_type -> filer_pb.TraverseBfsMetadataResponse
+	44, // 89: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse
+	44, // 90: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse
+	54, // 91: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse
+	56, // 92: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse
+	59, // 93: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:output_type -> filer_pb.CacheRemoteObjectToLocalClusterResponse
+	61, // 94: filer_pb.SeaweedFiler.DistributedLock:output_type -> filer_pb.LockResponse
+	63, // 95: filer_pb.SeaweedFiler.DistributedUnlock:output_type -> filer_pb.UnlockResponse
+	65, // 96: filer_pb.SeaweedFiler.FindLockOwner:output_type -> filer_pb.FindLockOwnerResponse
+	68, // 97: filer_pb.SeaweedFiler.TransferLocks:output_type -> filer_pb.TransferLocksResponse
+	70, // 98: filer_pb.SeaweedFiler.ReplicateLock:output_type -> filer_pb.ReplicateLockResponse
+	72, // [72:99] is the sub-list for method output_type
+	45, // [45:72] is the sub-list for method input_type
 	45, // [45:45] is the sub-list for extension type_name
 	45, // [45:45] is the sub-list for extension extendee
 	0,  // [0:45] is the sub-list for field type_name
@@ -5555,13 +5740,13 @@ func file_filer_proto_init() {
 	if File_filer_proto != nil {
 		return
 	}
-	file_filer_proto_msgTypes[67].OneofWrappers = []any{
+	file_filer_proto_msgTypes[69].OneofWrappers = []any{
 		(*StreamMutateEntryRequest_CreateRequest)(nil),
 		(*StreamMutateEntryRequest_UpdateRequest)(nil),
 		(*StreamMutateEntryRequest_DeleteRequest)(nil),
 		(*StreamMutateEntryRequest_RenameRequest)(nil),
 	}
-	file_filer_proto_msgTypes[68].OneofWrappers = []any{
+	file_filer_proto_msgTypes[70].OneofWrappers = []any{
 		(*StreamMutateEntryResponse_CreateResponse)(nil),
 		(*StreamMutateEntryResponse_UpdateResponse)(nil),
 		(*StreamMutateEntryResponse_DeleteResponse)(nil),
@@ -5573,7 +5758,7 @@ func file_filer_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: unsafe.Slice(unsafe.StringData(file_filer_proto_rawDesc), len(file_filer_proto_rawDesc)),
 			NumEnums:      2,
-			NumMessages:   74,
+			NumMessages:   76,
 			NumExtensions: 0,
 			NumServices:   1,
 		},

weed/pb/filer_pb/filer_grpc.pb.go

@@ -45,6 +45,7 @@ const (
 	SeaweedFiler_DistributedUnlock_FullMethodName               = "/filer_pb.SeaweedFiler/DistributedUnlock"
 	SeaweedFiler_FindLockOwner_FullMethodName                   = "/filer_pb.SeaweedFiler/FindLockOwner"
 	SeaweedFiler_TransferLocks_FullMethodName                   = "/filer_pb.SeaweedFiler/TransferLocks"
+	SeaweedFiler_ReplicateLock_FullMethodName                   = "/filer_pb.SeaweedFiler/ReplicateLock"
 )
 
 // SeaweedFilerClient is the client API for SeaweedFiler service.
@@ -78,6 +79,7 @@ type SeaweedFilerClient interface {
 	FindLockOwner(ctx context.Context, in *FindLockOwnerRequest, opts ...grpc.CallOption) (*FindLockOwnerResponse, error)
 	// distributed lock management internal use only
 	TransferLocks(ctx context.Context, in *TransferLocksRequest, opts ...grpc.CallOption) (*TransferLocksResponse, error)
+	ReplicateLock(ctx context.Context, in *ReplicateLockRequest, opts ...grpc.CallOption) (*ReplicateLockResponse, error)
 }
 
 type seaweedFilerClient struct {
@@ -396,6 +398,16 @@ func (c *seaweedFilerClient) TransferLocks(ctx context.Context, in *TransferLock
 	return out, nil
 }
 
+func (c *seaweedFilerClient) ReplicateLock(ctx context.Context, in *ReplicateLockRequest, opts ...grpc.CallOption) (*ReplicateLockResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ReplicateLockResponse)
+	err := c.cc.Invoke(ctx, SeaweedFiler_ReplicateLock_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // SeaweedFilerServer is the server API for SeaweedFiler service.
 // All implementations must embed UnimplementedSeaweedFilerServer
 // for forward compatibility.
@@ -427,6 +439,7 @@ type SeaweedFilerServer interface {
 	FindLockOwner(context.Context, *FindLockOwnerRequest) (*FindLockOwnerResponse, error)
 	// distributed lock management internal use only
 	TransferLocks(context.Context, *TransferLocksRequest) (*TransferLocksResponse, error)
+	ReplicateLock(context.Context, *ReplicateLockRequest) (*ReplicateLockResponse, error)
 	mustEmbedUnimplementedSeaweedFilerServer()
 }
 
@@ -515,6 +528,9 @@ func (UnimplementedSeaweedFilerServer) FindLockOwner(context.Context, *FindLockO
 func (UnimplementedSeaweedFilerServer) TransferLocks(context.Context, *TransferLocksRequest) (*TransferLocksResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method TransferLocks not implemented")
 }
+func (UnimplementedSeaweedFilerServer) ReplicateLock(context.Context, *ReplicateLockRequest) (*ReplicateLockResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ReplicateLock not implemented")
+}
 func (UnimplementedSeaweedFilerServer) mustEmbedUnimplementedSeaweedFilerServer() {}
 func (UnimplementedSeaweedFilerServer) testEmbeddedByValue()                      {}
 
@@ -958,6 +974,24 @@ func _SeaweedFiler_TransferLocks_Handler(srv interface{}, ctx context.Context, d
 	return interceptor(ctx, in, info, handler)
 }
 
+func _SeaweedFiler_ReplicateLock_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ReplicateLockRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedFilerServer).ReplicateLock(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedFiler_ReplicateLock_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedFilerServer).ReplicateLock(ctx, req.(*ReplicateLockRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 // SeaweedFiler_ServiceDesc is the grpc.ServiceDesc for SeaweedFiler service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -1045,6 +1079,10 @@ var SeaweedFiler_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "TransferLocks",
 			Handler:    _SeaweedFiler_TransferLocks_Handler,
 		},
+		{
+			MethodName: "ReplicateLock",
+			Handler:    _SeaweedFiler_ReplicateLock_Handler,
+		},
 	},
 	Streams: []grpc.StreamDesc{
 		{

weed/pb/filer_pb/filer_pb_helper.go

@@ -11,6 +11,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/viant/ptrie"
 	"google.golang.org/protobuf/proto"
 )
@@ -247,6 +248,119 @@ func IsRename(event *SubscribeMetadataResponse) bool {
 			event.EventNotification.NewEntry.Name != event.EventNotification.OldEntry.Name)
 }
 
+func MetadataEventSourceDirectory(event *SubscribeMetadataResponse) string {
+	if event == nil {
+		return ""
+	}
+	return event.Directory
+}
+
+func MetadataEventTargetDirectory(event *SubscribeMetadataResponse) string {
+	if event == nil {
+		return ""
+	}
+	if event.EventNotification != nil && event.EventNotification.NewParentPath != "" {
+		return event.EventNotification.NewParentPath
+	}
+	return event.Directory
+}
+
+func metadataEventSourceEntryName(event *SubscribeMetadataResponse) string {
+	if event == nil || event.EventNotification == nil {
+		return ""
+	}
+	if event.EventNotification.OldEntry != nil {
+		return event.EventNotification.OldEntry.Name
+	}
+	if event.EventNotification.NewEntry != nil {
+		return event.EventNotification.NewEntry.Name
+	}
+	return ""
+}
+
+func metadataEventTargetEntryName(event *SubscribeMetadataResponse) string {
+	if event == nil || event.EventNotification == nil {
+		return ""
+	}
+	if event.EventNotification.NewEntry != nil {
+		return event.EventNotification.NewEntry.Name
+	}
+	if event.EventNotification.OldEntry != nil {
+		return event.EventNotification.OldEntry.Name
+	}
+	return ""
+}
+
+func MetadataEventSourceFullPath(event *SubscribeMetadataResponse) string {
+	return util.Join(MetadataEventSourceDirectory(event), metadataEventSourceEntryName(event))
+}
+
+func MetadataEventTargetFullPath(event *SubscribeMetadataResponse) string {
+	return util.Join(MetadataEventTargetDirectory(event), metadataEventTargetEntryName(event))
+}
+
+func MetadataEventTouchesDirectory(event *SubscribeMetadataResponse, dir string) bool {
+	if MetadataEventSourceDirectory(event) == dir {
+		return true
+	}
+	return event != nil &&
+		event.EventNotification != nil &&
+		event.EventNotification.NewEntry != nil &&
+		MetadataEventTargetDirectory(event) == dir
+}
+
+func MetadataEventTouchesDirectoryPrefix(event *SubscribeMetadataResponse, prefix string) bool {
+	if strings.HasPrefix(MetadataEventSourceDirectory(event), prefix) {
+		return true
+	}
+	return event != nil &&
+		event.EventNotification != nil &&
+		event.EventNotification.NewEntry != nil &&
+		strings.HasPrefix(MetadataEventTargetDirectory(event), prefix)
+}
+
+func MetadataEventMatchesSubscription(event *SubscribeMetadataResponse, pathPrefix string, pathPrefixes []string, directories []string) bool {
+	if event == nil {
+		return false
+	}
+
+	if metadataEventMatchesPath(MetadataEventSourceFullPath(event), MetadataEventSourceDirectory(event), pathPrefix, pathPrefixes, directories) {
+		return true
+	}
+
+	return event.EventNotification != nil &&
+		event.EventNotification.NewEntry != nil &&
+		metadataEventMatchesPath(MetadataEventTargetFullPath(event), MetadataEventTargetDirectory(event), pathPrefix, pathPrefixes, directories)
+}
+
+func metadataEventMatchesPath(fullPath, dirPath, pathPrefix string, pathPrefixes []string, directories []string) bool {
+	if hasPrefixIn(fullPath, pathPrefixes) {
+		return true
+	}
+	if matchByDirectory(dirPath, directories) {
+		return true
+	}
+	return strings.HasPrefix(fullPath, pathPrefix)
+}
+
+func hasPrefixIn(text string, prefixes []string) bool {
+	for _, p := range prefixes {
+		if strings.HasPrefix(text, p) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchByDirectory(dirPath string, directories []string) bool {
+	for _, dir := range directories {
+		if dirPath == dir {
+			return true
+		}
+	}
+	return false
+}
+
 var _ = ptrie.KeyProvider(&FilerConf_PathConf{})
 
 func (fp *FilerConf_PathConf) Key() interface{} {

weed/pb/filer_pb/filer_pb_helper_test.go

@@ -15,3 +15,73 @@ func TestFileIdSize(t *testing.T) {
 	println(len(fileIdStr))
 	println(len(bytes))
 }
+
+func TestMetadataEventMatchesSubscription(t *testing.T) {
+	event := &SubscribeMetadataResponse{
+		Directory: "/tmp",
+		EventNotification: &EventNotification{
+			OldEntry:      &Entry{Name: "old-name"},
+			NewEntry:      &Entry{Name: "new-name"},
+			NewParentPath: "/watched",
+		},
+	}
+
+	tests := []struct {
+		name         string
+		pathPrefix   string
+		pathPrefixes []string
+		directories  []string
+	}{
+		{
+			name:       "primary path prefix matches rename target",
+			pathPrefix: "/watched/new-name",
+		},
+		{
+			name:         "additional path prefix matches rename target",
+			pathPrefix:   "/data",
+			pathPrefixes: []string{"/watched"},
+		},
+		{
+			name:        "directory watch matches rename target directory",
+			pathPrefix:  "/data",
+			directories: []string{"/watched"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if !MetadataEventMatchesSubscription(event, tt.pathPrefix, tt.pathPrefixes, tt.directories) {
+				t.Fatalf("MetadataEventMatchesSubscription returned false")
+			}
+		})
+	}
+}
+
+func TestMetadataEventTouchesDirectoryHelpers(t *testing.T) {
+	renameInto := &SubscribeMetadataResponse{
+		Directory: "/tmp",
+		EventNotification: &EventNotification{
+			OldEntry:      &Entry{Name: "filer.conf"},
+			NewEntry:      &Entry{Name: "filer.conf"},
+			NewParentPath: "/etc/seaweedfs",
+		},
+	}
+	if got := MetadataEventTargetDirectory(renameInto); got != "/etc/seaweedfs" {
+		t.Fatalf("MetadataEventTargetDirectory = %q, want /etc/seaweedfs", got)
+	}
+	if !MetadataEventTouchesDirectory(renameInto, "/etc/seaweedfs") {
+		t.Fatalf("expected rename target to touch /etc/seaweedfs")
+	}
+
+	renameOut := &SubscribeMetadataResponse{
+		Directory: "/etc/remote",
+		EventNotification: &EventNotification{
+			OldEntry:      &Entry{Name: "remote.conf"},
+			NewEntry:      &Entry{Name: "remote.conf"},
+			NewParentPath: "/tmp",
+		},
+	}
+	if !MetadataEventTouchesDirectoryPrefix(renameOut, "/etc/remote") {
+		t.Fatalf("expected rename source to touch /etc/remote")
+	}
+}

weed/pb/filer_pb_direct_read.go

@@ -255,53 +255,14 @@ func processOneLogEntry(logEntry *filer_pb.LogEntry, filter PathFilter, processE
 const systemLogDir = "/topics/.system/log"
 
 func matchesFilter(resp *filer_pb.SubscribeMetadataResponse, filter PathFilter) bool {
-	var entryName string
-	if resp.EventNotification != nil {
-		if resp.EventNotification.OldEntry != nil {
-			entryName = resp.EventNotification.OldEntry.Name
-		} else if resp.EventNotification.NewEntry != nil {
-			entryName = resp.EventNotification.NewEntry.Name
-		}
-	}
-
-	fullpath := util.Join(resp.Directory, entryName)
+	fullpath := filer_pb.MetadataEventSourceFullPath(resp)
 
 	// Skip internal meta log entries
 	if strings.HasPrefix(fullpath, systemLogDir) {
 		return false
 	}
 
-	// Check AdditionalPathPrefixes
-	for _, p := range filter.AdditionalPathPrefixes {
-		if strings.HasPrefix(fullpath, p) {
-			return true
-		}
-	}
-
-	// Check DirectoriesToWatch (exact directory match)
-	for _, dir := range filter.DirectoriesToWatch {
-		if resp.Directory == dir {
-			return true
-		}
-	}
-
-	// Check primary PathPrefix
-	if filter.PathPrefix == "" || filter.PathPrefix == "/" {
-		return true
-	}
-	if strings.HasPrefix(fullpath, filter.PathPrefix) {
-		return true
-	}
-
-	// Check rename target
-	if resp.EventNotification != nil && resp.EventNotification.NewParentPath != "" {
-		newFullPath := util.Join(resp.EventNotification.NewParentPath, entryName)
-		if strings.HasPrefix(newFullPath, filter.PathPrefix) {
-			return true
-		}
-	}
-
-	return false
+	return filer_pb.MetadataEventMatchesSubscription(resp, filter.PathPrefix, filter.AdditionalPathPrefixes, filter.DirectoriesToWatch)
 }
 
 // isChunkNotFound checks if an error indicates a missing volume chunk.
@@ -323,10 +284,10 @@ type logEntryHeapItem struct {
 
 type logEntryHeap []*logEntryHeapItem
 
-func (h logEntryHeap) Len() int            { return len(h) }
-func (h logEntryHeap) Less(i, j int) bool  { return h[i].entry.TsNs < h[j].entry.TsNs }
-func (h logEntryHeap) Swap(i, j int)       { h[i], h[j] = h[j], h[i] }
-func (h *logEntryHeap) Push(x any)         { *h = append(*h, x.(*logEntryHeapItem)) }
+func (h logEntryHeap) Len() int           { return len(h) }
+func (h logEntryHeap) Less(i, j int) bool { return h[i].entry.TsNs < h[j].entry.TsNs }
+func (h logEntryHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+func (h *logEntryHeap) Push(x any)        { *h = append(*h, x.(*logEntryHeapItem)) }
 func (h *logEntryHeap) Pop() any {
 	old := *h
 	n := len(old)

weed/pb/grpc_client_server.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"fmt"
 	"math/rand/v2"
+	"net"
 	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
@@ -42,6 +44,12 @@ var (
 	// cache grpc connections
 	grpcClients     = make(map[string]*versionedGrpcClient)
 	grpcClientsLock sync.Mutex
+
+	// localGrpcSockets maps gRPC port numbers to Unix socket paths.
+	// When registered (by mini mode), gRPC clients connect via Unix socket
+	// instead of TCP for local services.
+	localGrpcSockets     = make(map[int]string)
+	localGrpcSocketsLock sync.RWMutex
 )
 
 type versionedGrpcClient struct {
@@ -55,6 +63,59 @@ func init() {
 	http.DefaultTransport.(*http.Transport).MaxIdleConns = 1024
 }
 
+// RegisterLocalGrpcSocket registers a Unix socket path for a gRPC port.
+// When a gRPC client dials an address on this port, it uses the Unix socket.
+func RegisterLocalGrpcSocket(grpcPort int, socketPath string) {
+	localGrpcSocketsLock.Lock()
+	defer localGrpcSocketsLock.Unlock()
+	localGrpcSockets[grpcPort] = socketPath
+}
+
+// GetLocalGrpcSocket returns the Unix socket path for a gRPC port, or empty if not registered.
+func GetLocalGrpcSocket(grpcPort int) string {
+	localGrpcSocketsLock.RLock()
+	defer localGrpcSocketsLock.RUnlock()
+	return localGrpcSockets[grpcPort]
+}
+
+// resolveLocalGrpcSocket extracts the port from a gRPC address and returns
+// the registered Unix socket path, if any.
+func resolveLocalGrpcSocket(address string) string {
+	_, portStr, err := net.SplitHostPort(address)
+	if err != nil {
+		return ""
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		return ""
+	}
+	return GetLocalGrpcSocket(port)
+}
+
+// ServeGrpcOnLocalSocket starts serving a gRPC server on a Unix socket
+// if one is registered for the given port.
+func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) {
+	socketPath := GetLocalGrpcSocket(grpcPort)
+	if socketPath == "" {
+		return
+	}
+	if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
+		glog.Warningf("Failed to remove old gRPC socket %s: %v", socketPath, err)
+	}
+	listener, err := net.Listen("unix", socketPath)
+	if err != nil {
+		glog.Errorf("Failed to listen on gRPC Unix socket %s: %v", socketPath, err)
+		return
+	}
+	glog.V(0).Infof("gRPC also listening on Unix socket %s", socketPath)
+	go func() {
+		if err := grpcServer.Serve(listener); err != nil && err != grpc.ErrServerStopped {
+			glog.Errorf("gRPC Unix socket server error on %s: %v", socketPath, err)
+		}
+		os.Remove(socketPath)
+	}()
+}
+
 func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
 	var options []grpc.ServerOption
 	options = append(options,
@@ -97,6 +158,14 @@ func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
 func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
 	var options []grpc.DialOption
 
+	// Route through Unix socket if one is registered for this address's port
+	if socketPath := resolveLocalGrpcSocket(address); socketPath != "" {
+		options = append(options, grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) {
+			var d net.Dialer
+			return d.DialContext(ctx, "unix", socketPath)
+		}))
+	}
+
 	options = append(options,
 		grpc.WithDefaultCallOptions(
 			grpc.MaxCallSendMsgSize(Max_Message_Size),

weed/pb/master.proto

@@ -197,6 +197,17 @@ message ClusterNodeUpdate {
 message KeepConnectedResponse {
   VolumeLocation volume_location = 1;
   ClusterNodeUpdate cluster_node_update = 2;
+  LockRingUpdate lock_ring_update = 3;
+}
+
+// LockRingUpdate is sent by the master to all filers when the lock ring
+// membership changes. The master batches rapid changes (e.g., node drop + join)
+// and sends the complete member list atomically, avoiding intermediate ring
+// states that would cause unnecessary lock churn.
+message LockRingUpdate {
+  string filer_group = 1;
+  repeated string servers = 2;
+  int64 version = 3;
 }
 
 message LookupVolumeRequest {

weed/query/engine/broker_client.go

@@ -2,7 +2,6 @@ package engine
 
 import (
 	"context"
-	"encoding/binary"
 	"fmt"
 	"io"
 	"strings"
@@ -18,7 +17,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	jsonpb "google.golang.org/protobuf/encoding/protojson"
@@ -509,77 +507,3 @@ func (c *BrokerClient) GetUnflushedMessages(ctx context.Context, namespace, topi
 
 	return logEntries, nil
 }
-
-// getEarliestBufferStart finds the earliest buffer_start index from disk files in the partition
-//
-// This method handles three scenarios for seamless broker querying:
-// 1. Live log files exist: Uses their buffer_start metadata (most recent boundaries)
-// 2. Only Parquet files exist: Uses Parquet buffer_start metadata (preserved from archived sources)
-// 3. Mixed files: Uses earliest buffer_start from all sources for comprehensive coverage
-//
-// This ensures continuous real-time querying capability even after log file compaction/archival
-func (c *BrokerClient) getEarliestBufferStart(ctx context.Context, partitionPath string) (int64, error) {
-	filerClient, err := c.GetFilerClient()
-	if err != nil {
-		return 0, fmt.Errorf("failed to get filer client: %v", err)
-	}
-
-	var earliestBufferIndex int64 = -1 // -1 means no buffer_start found
-	var logFileCount, parquetFileCount int
-	var bufferStartSources []string // Track which files provide buffer_start
-
-	err = filer_pb.ReadDirAllEntries(ctx, filerClient, util.FullPath(partitionPath), "", func(entry *filer_pb.Entry, isLast bool) error {
-		// Skip directories
-		if entry.IsDirectory {
-			return nil
-		}
-
-		// Count file types for scenario detection
-		if strings.HasSuffix(entry.Name, ".parquet") {
-			parquetFileCount++
-		} else {
-			logFileCount++
-		}
-
-		// Extract buffer_start from file extended attributes (both log files and parquet files)
-		bufferStart := c.getBufferStartFromEntry(entry)
-		if bufferStart != nil && bufferStart.StartIndex > 0 {
-			if earliestBufferIndex == -1 || bufferStart.StartIndex < earliestBufferIndex {
-				earliestBufferIndex = bufferStart.StartIndex
-			}
-			bufferStartSources = append(bufferStartSources, entry.Name)
-		}
-
-		return nil
-	})
-
-	if err != nil {
-		return 0, fmt.Errorf("failed to scan partition directory: %v", err)
-	}
-
-	if earliestBufferIndex == -1 {
-		return 0, fmt.Errorf("no buffer_start metadata found in partition")
-	}
-
-	return earliestBufferIndex, nil
-}
-
-// getBufferStartFromEntry extracts LogBufferStart from file entry metadata
-// Only supports binary format (used by both log files and Parquet files)
-func (c *BrokerClient) getBufferStartFromEntry(entry *filer_pb.Entry) *LogBufferStart {
-	if entry.Extended == nil {
-		return nil
-	}
-
-	if startData, exists := entry.Extended["buffer_start"]; exists {
-		// Only support binary format
-		if len(startData) == 8 {
-			startIndex := int64(binary.BigEndian.Uint64(startData))
-			if startIndex > 0 {
-				return &LogBufferStart{StartIndex: startIndex}
-			}
-		}
-	}
-
-	return nil
-}

weed/query/engine/data_conversion.go

@@ -170,27 +170,6 @@ func (e *SQLEngine) convertRawValueToSchemaValue(rawValue interface{}) *schema_p
 	}
 }
 
-// convertJSONValueToSchemaValue converts JSON values to schema_pb.Value
-func (e *SQLEngine) convertJSONValueToSchemaValue(jsonValue interface{}) *schema_pb.Value {
-	switch v := jsonValue.(type) {
-	case string:
-		return &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: v}}
-	case float64:
-		// JSON numbers are always float64, try to detect if it's actually an integer
-		if v == float64(int64(v)) {
-			return &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: int64(v)}}
-		}
-		return &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: v}}
-	case bool:
-		return &schema_pb.Value{Kind: &schema_pb.Value_BoolValue{BoolValue: v}}
-	case nil:
-		return nil
-	default:
-		// Convert other types to string
-		return &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: fmt.Sprintf("%v", v)}}
-	}
-}
-
 // Helper functions for aggregation processing
 
 // isNullValue checks if a schema_pb.Value is null or empty

weed/query/engine/engine.go

@@ -2175,361 +2175,6 @@ func (e *SQLEngine) executeRegularSelectWithHybridScanner(ctx context.Context, h
 	return e.ConvertToSQLResultWithExpressions(hybridScanner, results, stmt.SelectExprs), nil
 }
 
-// executeSelectStatementWithBrokerStats handles SELECT queries with broker buffer statistics capture
-// This is used by EXPLAIN queries to capture complete data source information including broker memory
-func (e *SQLEngine) executeSelectStatementWithBrokerStats(ctx context.Context, stmt *SelectStatement, plan *QueryExecutionPlan) (*QueryResult, error) {
-	// Parse FROM clause to get table (topic) information
-	if len(stmt.From) != 1 {
-		err := fmt.Errorf("SELECT supports single table queries only")
-		return &QueryResult{Error: err}, err
-	}
-
-	// Extract table reference
-	var database, tableName string
-	switch table := stmt.From[0].(type) {
-	case *AliasedTableExpr:
-		switch tableExpr := table.Expr.(type) {
-		case TableName:
-			tableName = tableExpr.Name.String()
-			if tableExpr.Qualifier != nil && tableExpr.Qualifier.String() != "" {
-				database = tableExpr.Qualifier.String()
-			}
-		default:
-			err := fmt.Errorf("unsupported table expression: %T", tableExpr)
-			return &QueryResult{Error: err}, err
-		}
-	default:
-		err := fmt.Errorf("unsupported FROM clause: %T", table)
-		return &QueryResult{Error: err}, err
-	}
-
-	// Use current database context if not specified
-	if database == "" {
-		database = e.catalog.GetCurrentDatabase()
-		if database == "" {
-			database = "default"
-		}
-	}
-
-	// Auto-discover and register topic if not already in catalog
-	if _, err := e.catalog.GetTableInfo(database, tableName); err != nil {
-		// Topic not in catalog, try to discover and register it
-		if regErr := e.discoverAndRegisterTopic(ctx, database, tableName); regErr != nil {
-			// Return error immediately for non-existent topics instead of falling back to sample data
-			return &QueryResult{Error: regErr}, regErr
-		}
-	}
-
-	// Create HybridMessageScanner for the topic (reads both live logs + Parquet files)
-	// Get filerClient from broker connection (works with both real and mock brokers)
-	var filerClient filer_pb.FilerClient
-	var filerClientErr error
-	filerClient, filerClientErr = e.catalog.brokerClient.GetFilerClient()
-	if filerClientErr != nil {
-		// Return error if filer client is not available for topic access
-		return &QueryResult{Error: filerClientErr}, filerClientErr
-	}
-
-	hybridScanner, err := NewHybridMessageScanner(filerClient, e.catalog.brokerClient, database, tableName, e)
-	if err != nil {
-		// Handle quiet topics gracefully: topics exist but have no active schema/brokers
-		if IsNoSchemaError(err) {
-			// Return empty result for quiet topics (normal in production environments)
-			return &QueryResult{
-				Columns:  []string{},
-				Rows:     [][]sqltypes.Value{},
-				Database: database,
-				Table:    tableName,
-			}, nil
-		}
-		// Return error for other access issues (truly non-existent topics, etc.)
-		topicErr := fmt.Errorf("failed to access topic %s.%s: %v", database, tableName, err)
-		return &QueryResult{Error: topicErr}, topicErr
-	}
-
-	// Parse SELECT columns and detect aggregation functions
-	var columns []string
-	var aggregations []AggregationSpec
-	selectAll := false
-	hasAggregations := false
-	_ = hasAggregations // Used later in aggregation routing
-	// Track required base columns for arithmetic expressions
-	baseColumnsSet := make(map[string]bool)
-
-	for _, selectExpr := range stmt.SelectExprs {
-		switch expr := selectExpr.(type) {
-		case *StarExpr:
-			selectAll = true
-		case *AliasedExpr:
-			switch col := expr.Expr.(type) {
-			case *ColName:
-				colName := col.Name.String()
-				columns = append(columns, colName)
-				baseColumnsSet[colName] = true
-			case *ArithmeticExpr:
-				// Handle arithmetic expressions like id+user_id and string concatenation like name||suffix
-				columns = append(columns, e.getArithmeticExpressionAlias(col))
-				// Extract base columns needed for this arithmetic expression
-				e.extractBaseColumns(col, baseColumnsSet)
-			case *SQLVal:
-				// Handle string/numeric literals like 'good', 123, etc.
-				columns = append(columns, e.getSQLValAlias(col))
-			case *FuncExpr:
-				// Distinguish between aggregation functions and string functions
-				funcName := strings.ToUpper(col.Name.String())
-				if e.isAggregationFunction(funcName) {
-					// Handle aggregation functions
-					aggSpec, err := e.parseAggregationFunction(col, expr)
-					if err != nil {
-						return &QueryResult{Error: err}, err
-					}
-					aggregations = append(aggregations, *aggSpec)
-					hasAggregations = true
-				} else if e.isStringFunction(funcName) {
-					// Handle string functions like UPPER, LENGTH, etc.
-					columns = append(columns, e.getStringFunctionAlias(col))
-					// Extract base columns needed for this string function
-					e.extractBaseColumnsFromFunction(col, baseColumnsSet)
-				} else if e.isDateTimeFunction(funcName) {
-					// Handle datetime functions like CURRENT_DATE, NOW, EXTRACT, DATE_TRUNC
-					columns = append(columns, e.getDateTimeFunctionAlias(col))
-					// Extract base columns needed for this datetime function
-					e.extractBaseColumnsFromFunction(col, baseColumnsSet)
-				} else {
-					return &QueryResult{Error: fmt.Errorf("unsupported function: %s", funcName)}, fmt.Errorf("unsupported function: %s", funcName)
-				}
-			default:
-				err := fmt.Errorf("unsupported SELECT expression: %T", col)
-				return &QueryResult{Error: err}, err
-			}
-		default:
-			err := fmt.Errorf("unsupported SELECT expression: %T", expr)
-			return &QueryResult{Error: err}, err
-		}
-	}
-
-	// If we have aggregations, use aggregation query path
-	if hasAggregations {
-		return e.executeAggregationQuery(ctx, hybridScanner, aggregations, stmt)
-	}
-
-	// Parse WHERE clause for predicate pushdown
-	var predicate func(*schema_pb.RecordValue) bool
-	if stmt.Where != nil {
-		predicate, err = e.buildPredicateWithContext(stmt.Where.Expr, stmt.SelectExprs)
-		if err != nil {
-			return &QueryResult{Error: err}, err
-		}
-	}
-
-	// Parse LIMIT and OFFSET clauses
-	// Use -1 to distinguish "no LIMIT" from "LIMIT 0"
-	limit := -1
-	offset := 0
-	if stmt.Limit != nil && stmt.Limit.Rowcount != nil {
-		switch limitExpr := stmt.Limit.Rowcount.(type) {
-		case *SQLVal:
-			if limitExpr.Type == IntVal {
-				var parseErr error
-				limit64, parseErr := strconv.ParseInt(string(limitExpr.Val), 10, 64)
-				if parseErr != nil {
-					return &QueryResult{Error: parseErr}, parseErr
-				}
-				if limit64 > math.MaxInt32 || limit64 < 0 {
-					return &QueryResult{Error: fmt.Errorf("LIMIT value %d is out of valid range", limit64)}, fmt.Errorf("LIMIT value %d is out of valid range", limit64)
-				}
-				limit = int(limit64)
-			}
-		}
-	}
-
-	// Parse OFFSET clause if present
-	if stmt.Limit != nil && stmt.Limit.Offset != nil {
-		switch offsetExpr := stmt.Limit.Offset.(type) {
-		case *SQLVal:
-			if offsetExpr.Type == IntVal {
-				var parseErr error
-				offset64, parseErr := strconv.ParseInt(string(offsetExpr.Val), 10, 64)
-				if parseErr != nil {
-					return &QueryResult{Error: parseErr}, parseErr
-				}
-				if offset64 > math.MaxInt32 || offset64 < 0 {
-					return &QueryResult{Error: fmt.Errorf("OFFSET value %d is out of valid range", offset64)}, fmt.Errorf("OFFSET value %d is out of valid range", offset64)
-				}
-				offset = int(offset64)
-			}
-		}
-	}
-
-	// Build hybrid scan options
-	// Extract time filters from WHERE clause to optimize scanning
-	startTimeNs, stopTimeNs := int64(0), int64(0)
-	if stmt.Where != nil {
-		startTimeNs, stopTimeNs = e.extractTimeFilters(stmt.Where.Expr)
-	}
-
-	hybridScanOptions := HybridScanOptions{
-		StartTimeNs: startTimeNs, // Extracted from WHERE clause time comparisons
-		StopTimeNs:  stopTimeNs,  // Extracted from WHERE clause time comparisons
-		Limit:       limit,
-		Offset:      offset,
-		Predicate:   predicate,
-	}
-
-	if !selectAll {
-		// Convert baseColumnsSet to slice for hybrid scan options
-		baseColumns := make([]string, 0, len(baseColumnsSet))
-		for columnName := range baseColumnsSet {
-			baseColumns = append(baseColumns, columnName)
-		}
-		// Use base columns (not expression aliases) for data retrieval
-		if len(baseColumns) > 0 {
-			hybridScanOptions.Columns = baseColumns
-		} else {
-			// If no base columns found (shouldn't happen), use original columns
-			hybridScanOptions.Columns = columns
-		}
-	}
-
-	// Execute the hybrid scan with stats capture for EXPLAIN
-	var results []HybridScanResult
-	if plan != nil {
-		// EXPLAIN mode - capture broker buffer stats
-		var stats *HybridScanStats
-		results, stats, err = hybridScanner.ScanWithStats(ctx, hybridScanOptions)
-		if err != nil {
-			return &QueryResult{Error: err}, err
-		}
-
-		// Populate plan with broker buffer information
-		if stats != nil {
-			plan.BrokerBufferQueried = stats.BrokerBufferQueried
-			plan.BrokerBufferMessages = stats.BrokerBufferMessages
-			plan.BufferStartIndex = stats.BufferStartIndex
-
-			// Add broker_buffer to data sources if buffer was queried
-			if stats.BrokerBufferQueried {
-				// Check if broker_buffer is already in data sources
-				hasBrokerBuffer := false
-				for _, source := range plan.DataSources {
-					if source == "broker_buffer" {
-						hasBrokerBuffer = true
-						break
-					}
-				}
-				if !hasBrokerBuffer {
-					plan.DataSources = append(plan.DataSources, "broker_buffer")
-				}
-			}
-		}
-
-		// Populate execution plan details with source file information for Data Sources Tree
-		if partitions, discoverErr := e.discoverTopicPartitions(database, tableName); discoverErr == nil {
-			// Add partition paths to execution plan details
-			plan.Details["partition_paths"] = partitions
-			// Persist time filter details for downstream pruning/diagnostics
-			plan.Details[PlanDetailStartTimeNs] = startTimeNs
-			plan.Details[PlanDetailStopTimeNs] = stopTimeNs
-
-			// Collect actual file information for each partition
-			var parquetFiles []string
-			var liveLogFiles []string
-			parquetSources := make(map[string]bool)
-
-			var parquetReadErrors []string
-			var liveLogListErrors []string
-			for _, partitionPath := range partitions {
-				// Get parquet files for this partition
-				if parquetStats, err := hybridScanner.ReadParquetStatistics(partitionPath); err == nil {
-					// Prune files by time range with debug logging
-					filteredStats := pruneParquetFilesByTime(ctx, parquetStats, hybridScanner, startTimeNs, stopTimeNs)
-
-					// Further prune by column statistics from WHERE clause
-					if stmt.Where != nil {
-						beforeColumnPrune := len(filteredStats)
-						filteredStats = e.pruneParquetFilesByColumnStats(ctx, filteredStats, stmt.Where.Expr)
-						columnPrunedCount := beforeColumnPrune - len(filteredStats)
-
-						if columnPrunedCount > 0 {
-							// Track column statistics optimization
-							if !contains(plan.OptimizationsUsed, "column_statistics_pruning") {
-								plan.OptimizationsUsed = append(plan.OptimizationsUsed, "column_statistics_pruning")
-							}
-						}
-					}
-					for _, stats := range filteredStats {
-						parquetFiles = append(parquetFiles, fmt.Sprintf("%s/%s", partitionPath, stats.FileName))
-					}
-				} else {
-					parquetReadErrors = append(parquetReadErrors, fmt.Sprintf("%s: %v", partitionPath, err))
-				}
-
-				// Merge accurate parquet sources from metadata
-				if sources, err := e.getParquetSourceFilesFromMetadata(partitionPath); err == nil {
-					for src := range sources {
-						parquetSources[src] = true
-					}
-				}
-
-				// Get live log files for this partition
-				if liveFiles, err := e.collectLiveLogFileNames(hybridScanner.filerClient, partitionPath); err == nil {
-					for _, fileName := range liveFiles {
-						// Exclude live log files that have been converted to parquet (deduplicated)
-						if parquetSources[fileName] {
-							continue
-						}
-						liveLogFiles = append(liveLogFiles, fmt.Sprintf("%s/%s", partitionPath, fileName))
-					}
-				} else {
-					liveLogListErrors = append(liveLogListErrors, fmt.Sprintf("%s: %v", partitionPath, err))
-				}
-			}
-
-			if len(parquetFiles) > 0 {
-				plan.Details["parquet_files"] = parquetFiles
-			}
-			if len(liveLogFiles) > 0 {
-				plan.Details["live_log_files"] = liveLogFiles
-			}
-			if len(parquetReadErrors) > 0 {
-				plan.Details["error_parquet_statistics"] = parquetReadErrors
-			}
-			if len(liveLogListErrors) > 0 {
-				plan.Details["error_live_log_listing"] = liveLogListErrors
-			}
-
-			// Update scan statistics for execution plan display
-			plan.PartitionsScanned = len(partitions)
-			plan.ParquetFilesScanned = len(parquetFiles)
-			plan.LiveLogFilesScanned = len(liveLogFiles)
-		} else {
-			// Handle partition discovery error
-			plan.Details["error_partition_discovery"] = discoverErr.Error()
-		}
-	} else {
-		// Normal mode - just get results
-		results, err = hybridScanner.Scan(ctx, hybridScanOptions)
-		if err != nil {
-			return &QueryResult{Error: err}, err
-		}
-	}
-
-	// Convert to SQL result format
-	if selectAll {
-		if len(columns) > 0 {
-			// SELECT *, specific_columns - include both auto-discovered and explicit columns
-			return hybridScanner.ConvertToSQLResultWithMixedColumns(results, columns), nil
-		} else {
-			// SELECT * only - let converter determine all columns (excludes system columns)
-			columns = nil
-			return hybridScanner.ConvertToSQLResult(results, columns), nil
-		}
-	}
-
-	// Handle custom column expressions (including arithmetic)
-	return e.ConvertToSQLResultWithExpressions(hybridScanner, results, stmt.SelectExprs), nil
-}
-
 // extractTimeFilters extracts time range filters from WHERE clause for optimization
 // This allows push-down of time-based queries to improve scan performance
 // Returns (startTimeNs, stopTimeNs) where 0 means unbounded

weed/query/engine/engine_test.go

@@ -1254,69 +1254,6 @@ func TestSQLEngine_LogBufferDeduplication_ServerRestartScenario(t *testing.T) {
 	// prevent false positive duplicates across server restarts
 }
 
-func TestBrokerClient_BinaryBufferStartFormat(t *testing.T) {
-	// Test scenario: getBufferStartFromEntry should only support binary format
-	// This tests the standardized binary format for buffer_start metadata
-	realBrokerClient := &BrokerClient{}
-
-	// Test binary format (used by both log files and Parquet files)
-	binaryEntry := &filer_pb.Entry{
-		Name:        "2025-01-07-14-30-45",
-		IsDirectory: false,
-		Extended: map[string][]byte{
-			"buffer_start": func() []byte {
-				// Binary format: 8-byte BigEndian
-				buf := make([]byte, 8)
-				binary.BigEndian.PutUint64(buf, uint64(2000001))
-				return buf
-			}(),
-		},
-	}
-
-	bufferStart := realBrokerClient.getBufferStartFromEntry(binaryEntry)
-	assert.NotNil(t, bufferStart)
-	assert.Equal(t, int64(2000001), bufferStart.StartIndex, "Should parse binary buffer_start metadata")
-
-	// Test Parquet file (same binary format)
-	parquetEntry := &filer_pb.Entry{
-		Name:        "2025-01-07-14-30.parquet",
-		IsDirectory: false,
-		Extended: map[string][]byte{
-			"buffer_start": func() []byte {
-				buf := make([]byte, 8)
-				binary.BigEndian.PutUint64(buf, uint64(1500001))
-				return buf
-			}(),
-		},
-	}
-
-	bufferStart = realBrokerClient.getBufferStartFromEntry(parquetEntry)
-	assert.NotNil(t, bufferStart)
-	assert.Equal(t, int64(1500001), bufferStart.StartIndex, "Should parse binary buffer_start from Parquet file")
-
-	// Test missing metadata
-	emptyEntry := &filer_pb.Entry{
-		Name:        "no-metadata",
-		IsDirectory: false,
-		Extended:    nil,
-	}
-
-	bufferStart = realBrokerClient.getBufferStartFromEntry(emptyEntry)
-	assert.Nil(t, bufferStart, "Should return nil for entry without buffer_start metadata")
-
-	// Test invalid format (wrong size)
-	invalidEntry := &filer_pb.Entry{
-		Name:        "invalid-metadata",
-		IsDirectory: false,
-		Extended: map[string][]byte{
-			"buffer_start": []byte("invalid"),
-		},
-	}
-
-	bufferStart = realBrokerClient.getBufferStartFromEntry(invalidEntry)
-	assert.Nil(t, bufferStart, "Should return nil for invalid buffer_start metadata")
-}
-
 // TestGetSQLValAlias tests the getSQLValAlias function, particularly for SQL injection prevention
 func TestGetSQLValAlias(t *testing.T) {
 	engine := &SQLEngine{}

weed/query/engine/hybrid_message_scanner.go

@@ -320,12 +320,6 @@ func (hms *HybridMessageScanner) ScanWithStats(ctx context.Context, options Hybr
 	return results, stats, nil
 }
 
-// scanUnflushedData queries brokers for unflushed in-memory data using buffer_start deduplication
-func (hms *HybridMessageScanner) scanUnflushedData(ctx context.Context, partition topic.Partition, options HybridScanOptions) ([]HybridScanResult, error) {
-	results, _, err := hms.scanUnflushedDataWithStats(ctx, partition, options)
-	return results, err
-}
-
 // scanUnflushedDataWithStats queries brokers for unflushed data and returns statistics
 func (hms *HybridMessageScanner) scanUnflushedDataWithStats(ctx context.Context, partition topic.Partition, options HybridScanOptions) ([]HybridScanResult, *HybridScanStats, error) {
 	var results []HybridScanResult
@@ -436,27 +430,6 @@ func (hms *HybridMessageScanner) scanUnflushedDataWithStats(ctx context.Context,
 	return results, stats, nil
 }
 
-// convertDataMessageToRecord converts mq_pb.DataMessage to schema_pb.RecordValue
-func (hms *HybridMessageScanner) convertDataMessageToRecord(msg *mq_pb.DataMessage) (*schema_pb.RecordValue, string, error) {
-	// Parse the message data as RecordValue
-	recordValue := &schema_pb.RecordValue{}
-	if err := proto.Unmarshal(msg.Value, recordValue); err != nil {
-		return nil, "", fmt.Errorf("failed to unmarshal message data: %v", err)
-	}
-
-	// Add system columns
-	if recordValue.Fields == nil {
-		recordValue.Fields = make(map[string]*schema_pb.Value)
-	}
-
-	// Add timestamp
-	recordValue.Fields[SW_COLUMN_NAME_TIMESTAMP] = &schema_pb.Value{
-		Kind: &schema_pb.Value_Int64Value{Int64Value: msg.TsNs},
-	}
-
-	return recordValue, string(msg.Key), nil
-}
-
 // discoverTopicPartitions discovers the actual partitions for this topic by scanning the filesystem
 // This finds real partition directories like v2025-09-01-07-16-34/0000-0630/
 func (hms *HybridMessageScanner) discoverTopicPartitions(ctx context.Context) ([]topic.Partition, error) {
@@ -521,15 +494,6 @@ func (hms *HybridMessageScanner) discoverTopicPartitions(ctx context.Context) ([
 	return allPartitions, nil
 }
 
-// scanPartitionHybrid scans a specific partition using the hybrid approach
-// This is where the magic happens - seamlessly reading ALL data sources:
-// 1. Unflushed in-memory data from brokers (REAL-TIME)
-// 2. Live logs + Parquet files from disk (FLUSHED/ARCHIVED)
-func (hms *HybridMessageScanner) scanPartitionHybrid(ctx context.Context, partition topic.Partition, options HybridScanOptions) ([]HybridScanResult, error) {
-	results, _, err := hms.scanPartitionHybridWithStats(ctx, partition, options)
-	return results, err
-}
-
 // scanPartitionHybridWithStats scans a specific partition using streaming merge for memory efficiency
 // PERFORMANCE IMPROVEMENT: Uses heap-based streaming merge instead of collecting all data and sorting
 // - Memory usage: O(k) where k = number of data sources, instead of O(n) where n = total records
@@ -647,23 +611,6 @@ func (hms *HybridMessageScanner) countLiveLogFiles(partition topic.Partition) (i
 	return fileCount, nil
 }
 
-// isControlEntry checks if a log entry is a control entry without actual data
-// Based on MQ system analysis, control entries are:
-// 1. DataMessages with populated Ctrl field (publisher close signals)
-// 2. Entries with empty keys (as filtered by subscriber)
-// NOTE: Messages with empty data but valid keys (like NOOP messages) are NOT control entries
-func (hms *HybridMessageScanner) isControlEntry(logEntry *filer_pb.LogEntry) bool {
-	// Pre-decode DataMessage if needed
-	var dataMessage *mq_pb.DataMessage
-	if len(logEntry.Data) > 0 {
-		dataMessage = &mq_pb.DataMessage{}
-		if err := proto.Unmarshal(logEntry.Data, dataMessage); err != nil {
-			dataMessage = nil // Failed to decode, treat as raw data
-		}
-	}
-	return hms.isControlEntryWithDecoded(logEntry, dataMessage)
-}
-
 // isControlEntryWithDecoded checks if a log entry is a control entry using pre-decoded DataMessage
 // This avoids duplicate protobuf unmarshaling when the DataMessage is already decoded
 func (hms *HybridMessageScanner) isControlEntryWithDecoded(logEntry *filer_pb.LogEntry, dataMessage *mq_pb.DataMessage) bool {
@@ -682,26 +629,6 @@ func (hms *HybridMessageScanner) isControlEntryWithDecoded(logEntry *filer_pb.Lo
 	return false
 }
 
-// isNullOrEmpty checks if a schema_pb.Value is null or empty
-func isNullOrEmpty(value *schema_pb.Value) bool {
-	if value == nil {
-		return true
-	}
-
-	switch v := value.Kind.(type) {
-	case *schema_pb.Value_StringValue:
-		return v.StringValue == ""
-	case *schema_pb.Value_BytesValue:
-		return len(v.BytesValue) == 0
-	case *schema_pb.Value_ListValue:
-		return v.ListValue == nil || len(v.ListValue.Values) == 0
-	case nil:
-		return true // No kind set means null
-	default:
-		return false
-	}
-}
-
 // isSchemaless checks if the scanner is configured for a schema-less topic
 // Schema-less topics only have system fields: _ts_ns, _key, and _value
 func (hms *HybridMessageScanner) isSchemaless() bool {
@@ -736,61 +663,6 @@ func (hms *HybridMessageScanner) isSchemaless() bool {
 	return hasValue && dataFieldCount == 1
 }
 
-// convertLogEntryToRecordValue converts a filer_pb.LogEntry to schema_pb.RecordValue
-// This handles both:
-// 1. Live log entries (raw message format)
-// 2. Parquet entries (already in schema_pb.RecordValue format)
-// 3. Schema-less topics (raw bytes in _value field)
-func (hms *HybridMessageScanner) convertLogEntryToRecordValue(logEntry *filer_pb.LogEntry) (*schema_pb.RecordValue, string, error) {
-	// For schema-less topics, put raw data directly into _value field
-	if hms.isSchemaless() {
-		recordValue := &schema_pb.RecordValue{
-			Fields: make(map[string]*schema_pb.Value),
-		}
-		recordValue.Fields[SW_COLUMN_NAME_TIMESTAMP] = &schema_pb.Value{
-			Kind: &schema_pb.Value_Int64Value{Int64Value: logEntry.TsNs},
-		}
-		recordValue.Fields[SW_COLUMN_NAME_KEY] = &schema_pb.Value{
-			Kind: &schema_pb.Value_BytesValue{BytesValue: logEntry.Key},
-		}
-		recordValue.Fields[SW_COLUMN_NAME_VALUE] = &schema_pb.Value{
-			Kind: &schema_pb.Value_BytesValue{BytesValue: logEntry.Data},
-		}
-		return recordValue, "live_log", nil
-	}
-
-	// Try to unmarshal as RecordValue first (Parquet format)
-	recordValue := &schema_pb.RecordValue{}
-	if err := proto.Unmarshal(logEntry.Data, recordValue); err == nil {
-		// This is an archived message from Parquet files
-		// FIX: Add system columns from LogEntry to RecordValue
-		if recordValue.Fields == nil {
-			recordValue.Fields = make(map[string]*schema_pb.Value)
-		}
-
-		// Add system columns from LogEntry
-		recordValue.Fields[SW_COLUMN_NAME_TIMESTAMP] = &schema_pb.Value{
-			Kind: &schema_pb.Value_Int64Value{Int64Value: logEntry.TsNs},
-		}
-		recordValue.Fields[SW_COLUMN_NAME_KEY] = &schema_pb.Value{
-			Kind: &schema_pb.Value_BytesValue{BytesValue: logEntry.Key},
-		}
-
-		return recordValue, "parquet_archive", nil
-	}
-
-	// If not a RecordValue, this is raw live message data - parse with schema
-	return hms.parseRawMessageWithSchema(logEntry)
-}
-
-// min returns the minimum of two integers
-func min(a, b int) int {
-	if a < b {
-		return a
-	}
-	return b
-}
-
 // parseRawMessageWithSchema parses raw live message data using the topic's schema
 // This provides proper type conversion and field mapping instead of treating everything as strings
 func (hms *HybridMessageScanner) parseRawMessageWithSchema(logEntry *filer_pb.LogEntry) (*schema_pb.RecordValue, string, error) {