test(remote_storage/s3): assert WriteFile success in mime tests

filer.remote.sync was uploading every object without a Content-Type, so
S3-compatible backends (e.g. Backblaze B2) stored binary/octet-stream
and browsers refused to render HTML, CSS, etc.

Pass entry.Attributes.Mime through to UploadInput.ContentType, leaving
the header unset when no Mime is recorded so the remote keeps its own
default behavior.

.github/workflows/container_dev.yml

@@ -128,14 +128,14 @@ jobs:
 
       - name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}

.github/workflows/container_foundationdb_version.yml

@@ -133,7 +133,7 @@ jobs:
 
       - name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/container_latest.yml

@@ -221,13 +221,13 @@ jobs:
           buildkitd-config: /tmp/buildkitd.toml
       - name: Login to Docker Hub
         if: needs.setup.outputs.publish == 'true'
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Login to GHCR
         if: needs.setup.outputs.publish == 'true'
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
@@ -275,7 +275,7 @@ jobs:
           fi
       - name: Login to GHCR
         if: needs.setup.outputs.publish == 'true'
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
@@ -430,12 +430,12 @@ jobs:
             ghcr.io/chrislusf/seaweedfs
           tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
       - name: Login to Docker Hub
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Login to GHCR
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}

.github/workflows/container_release_foundationdb.yml

@@ -50,7 +50,7 @@ jobs:
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/container_release_unified.yml

@@ -237,14 +237,14 @@ jobs:
           
       - name: Login to Docker Hub
         if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           
       - name: Login to GHCR
         if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
@@ -300,14 +300,14 @@ jobs:
     steps:
       - name: Login to Docker Hub
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           
       - name: Login to GHCR
         if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
@@ -380,7 +380,7 @@ jobs:
             variant: large_disk
     steps:
       - name: Login to GHCR
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
@@ -429,13 +429,13 @@ jobs:
             latest_tag: latest_large_disk
     steps:
       - name: Login to Docker Hub
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to GHCR
-        uses: docker/login-action@v4.1.0
+        uses: docker/login-action@v4.2.0
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}

.github/workflows/container_rocksdb_version.yml

@@ -88,7 +88,7 @@ jobs:
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
 
       - name: Login to Docker Hub
-        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v1
+        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/ec-integration.yml

@@ -1,49 +0,0 @@
-name: EC Integration Tests
-
-on:
-  push:
-    branches: [ master ]
-    paths:
-      - 'weed/admin/**'
-      - 'weed/worker/**'
-      - 'test/erasure_coding/admin_dockertest/**'
-      - '.github/workflows/ec-integration.yml'
-  pull_request:
-    branches: [ master ]
-    paths:
-      - 'weed/admin/**'
-      - 'weed/worker/**'
-      - 'test/erasure_coding/admin_dockertest/**'
-      - '.github/workflows/ec-integration.yml'
-
-jobs:
-  ec-integration-test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v6
-      
-    - name: Set up Go
-      uses: actions/setup-go@v6
-      with:
-        go-version-file: 'go.mod'
-        
-    - name: Build weed binary
-      run: |
-        cd weed
-        go build -o ../weed_bin
-        
-    - name: Run EC integration tests
-      run: |
-        cd test/erasure_coding/admin_dockertest
-        go test -v -timeout 15m ec_integration_test.go
-        
-    - name: Upload test logs on failure
-      if: failure()
-      uses: actions/upload-artifact@v7
-      with:
-        name: ec-test-logs
-        path: test/erasure_coding/admin_dockertest/tmp/logs/
-        retention-days: 7

.github/workflows/helm_ci.yml

@@ -68,6 +68,135 @@ jobs:
           grep -q "security-config" /tmp/security.yaml
           echo "✓ Security configuration renders correctly"
 
+          echo ""
+          echo "=== Testing IAM gRPC opt-in path ==="
+          # Regression test: the filer registers the IAM gRPC service the
+          # Admin UI Users tab calls only when jwt.filer_signing.key is in
+          # security.toml. Operators must be able to enable that without
+          # the cert-manager mTLS bundle.
+          # Install PyYAML explicitly: this block runs before the later
+          # security+S3 block that does the same install, and we don't
+          # want to rely on the runner image shipping it.
+          pip install pyyaml -q
+          python3 - "$CHART_DIR" <<'PYEOF'
+          import subprocess, sys, yaml
+          chart = sys.argv[1]
+
+          def render(values):
+              args = ["helm", "template", "test", chart]
+              for k, v in values.items():
+                  args += ["--set", f"{k}={v}"]
+              return subprocess.check_output(args, text=True)
+
+          def docs(manifest):
+              return [d for d in yaml.safe_load_all(manifest) if d]
+
+          def configmap(manifest, name):
+              for d in docs(manifest):
+                  if d.get("kind") == "ConfigMap" and d["metadata"]["name"] == name:
+                      return d
+              return None
+
+          def workload_mounts(manifest, name):
+              for d in docs(manifest):
+                  if d.get("kind") not in ("Deployment", "StatefulSet"):
+                      continue
+                  if d["metadata"]["name"] != name:
+                      continue
+                  pod = d["spec"]["template"]["spec"]
+                  vols = {v["name"] for v in pod.get("volumes", [])}
+                  mounts = set()
+                  for c in pod.get("containers", []):
+                      for vm in c.get("volumeMounts", []):
+                          mounts.add(vm["name"])
+                  return vols, mounts
+              return None, None
+
+          failed = []
+
+          # Case 1: defaults. The chart historically rendered nothing
+          # security-related; preserve that so this PR is non-breaking on
+          # existing installs.
+          out = render({})
+          if configmap(out, "test-seaweedfs-security-config") is not None:
+              failed.append("defaults: security ConfigMap should not render")
+          else:
+              print("✓ defaults: no security-config ConfigMap (unchanged)")
+
+          # Case 2: filerWrite=true alone is the documented opt-in for
+          # the Admin UI Users tab. Configmap must render with
+          # [jwt.filer_signing] and NO [grpc.*] sections (cert paths
+          # only exist with mTLS).
+          out = render({
+              "global.seaweedfs.securityConfig.jwtSigning.filerWrite": "true",
+              "admin.enabled": "true",
+          })
+          cm = configmap(out, "test-seaweedfs-security-config")
+          if cm is None:
+              failed.append("filerWrite=true: security ConfigMap missing")
+          else:
+              toml = cm["data"]["security.toml"]
+              if "[jwt.filer_signing]" not in toml:
+                  failed.append("filerWrite=true: security.toml missing [jwt.filer_signing]")
+              if "[grpc" in toml:
+                  failed.append("filerWrite=true: security.toml unexpectedly has [grpc.*] (would need cert mounts)")
+              if "[jwt.filer_signing]" in toml and "[grpc" not in toml:
+                  print("✓ filerWrite=true: security.toml has [jwt.filer_signing], no [grpc.*]")
+
+          # Case 3: filer + admin pods must MOUNT the security ConfigMap
+          # under filerWrite=true so the JWT key reaches both processes.
+          # Cert volumes must NOT be present (no mTLS).
+          for wl in ("test-seaweedfs-filer", "test-seaweedfs-admin"):
+              vols, mounts = workload_mounts(out, wl)
+              if vols is None:
+                  failed.append(f"filerWrite=true: workload {wl} not found")
+                  continue
+              if "security-config" not in vols or "security-config" not in mounts:
+                  failed.append(f"filerWrite=true: {wl} does not mount security-config (IAM gRPC would still fail)")
+              else:
+                  print(f"✓ filerWrite=true: {wl} mounts security-config")
+              cert_vols = {v for v in vols if v.endswith("-cert")}
+              if cert_vols:
+                  failed.append(f"filerWrite=true: {wl} unexpectedly has cert volumes {sorted(cert_vols)}")
+
+          # Case 4: enableSecurity=true must still render the full toml
+          # with both [jwt.signing] and [grpc.*]. Guards against the
+          # decoupling change accidentally regressing the mTLS path.
+          out = render({"global.seaweedfs.enableSecurity": "true"})
+          cm = configmap(out, "test-seaweedfs-security-config")
+          if cm is None:
+              failed.append("enableSecurity=true: security ConfigMap missing")
+          else:
+              toml = cm["data"]["security.toml"]
+              missing = [s for s in ("[jwt.signing]", "[grpc.master]") if s not in toml]
+              if missing:
+                  failed.append(f"enableSecurity=true: security.toml missing {missing}")
+              else:
+                  print("✓ enableSecurity=true: security.toml has [jwt.signing] + [grpc.*] preserved")
+
+          # Case 5: helper must tolerate explicit nulls (gemini-code-assist
+          # PR review). securityConfig=null was the parens-pattern crash
+          # the helper review caught.
+          for null_path in ("global.seaweedfs.securityConfig",
+                            "global.seaweedfs.securityConfig.jwtSigning"):
+              try:
+                  out = render({null_path: "null"})
+              except subprocess.CalledProcessError as e:
+                  failed.append(f"{null_path}=null: render failed: {e.output[:200] if e.output else e}")
+                  continue
+              if configmap(out, "test-seaweedfs-security-config") is not None:
+                  failed.append(f"{null_path}=null: should not render configmap")
+              else:
+                  print(f"✓ {null_path}=null: render tolerates explicit null")
+
+          if failed:
+              print("\nFAIL:", file=sys.stderr)
+              for f in failed:
+                  print(f"  - {f}", file=sys.stderr)
+              sys.exit(1)
+          PYEOF
+          echo "✓ IAM gRPC decoupling tests passed"
+
           echo "=== Testing with monitoring enabled ==="
           helm template test $CHART_DIR \
             --set global.seaweedfs.monitoring.enabled=true \

.github/workflows/s3-go-tests.yml

@@ -257,6 +257,7 @@ jobs:
         # TTL-pinned bucket collections piled up in a single run.
         test:
           - TestLifecycleAbortIncompleteMultipartUpload
+          - TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort
           - TestLifecycleBootstrapWalkOnExistingObjects
           - TestLifecycleConfigUpdateBetweenSweeps
           - TestLifecycleDeleteBucketLifecycleStopsDispatching

.github/workflows/s3-proxy-signature-tests.yml

@@ -71,8 +71,10 @@ jobs:
           echo "Waiting for SeaweedFS S3 gateway to be ready via proxy..."
           S3_READY=0
           for i in $(seq 1 30); do
-            # Check logs first for startup message (weed mini says "S3 service is ready")
-            if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)"; then
+            # Check logs first for the readiness line. weed mini's progress
+            # board prints "  S3           ready (Xs)"; older builds and the
+            # standalone S3 binary log "S3 (gateway|service) ... ready".
+            if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)|S3[[:space:]]+ready"; then
               echo "SeaweedFS S3 gateway is ready"
               S3_READY=1
               break

.github/workflows/s3-sdk-v2-routing-tests.yml

@@ -0,0 +1,110 @@
+name: "S3 SDK V2 Route Disambiguation Tests"
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - 'weed/s3api/**'
+      - 'test/s3/sdk_v2_routing/**'
+      - '.github/workflows/s3-sdk-v2-routing-tests.yml'
+  pull_request:
+    branches: [ master ]
+    paths:
+      - 'weed/s3api/**'
+      - 'test/s3/sdk_v2_routing/**'
+      - '.github/workflows/s3-sdk-v2-routing-tests.yml'
+
+concurrency:
+  group: ${{ github.head_ref || github.ref }}/s3-sdk-v2-routing-tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  s3-sdk-v2-routing-tests:
+    name: S3 SDK V2 Routing Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Install SeaweedFS
+        run: |
+          cd weed && go install -buildvcs=false
+
+      - name: Start weed mini (S3 on :8333)
+        # Pins the regression for issue #9559: AWS SDK V2 / Hadoop s3a
+        # listing a bucket literally named "buckets" must get an XML
+        # ListObjectsV2 response, not the JSON ListTableBuckets body
+        # served by the S3 Tables REST endpoint on the same path.
+        run: |
+          mkdir -p /tmp/seaweedfs-sdk-v2-routing
+          cat > /tmp/seaweedfs-sdk-v2-routing-s3.json <<'JSON'
+          {
+            "identities": [
+              {
+                "name": "admin",
+                "credentials": [
+                  {"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
+                ],
+                "actions": ["Admin", "Read", "Write"]
+              }
+            ]
+          }
+          JSON
+          AWS_ACCESS_KEY_ID=some_access_key1 \
+          AWS_SECRET_ACCESS_KEY=some_secret_key1 \
+            weed mini \
+              -dir=/tmp/seaweedfs-sdk-v2-routing \
+              -s3.port=8333 \
+              -s3.config=/tmp/seaweedfs-sdk-v2-routing-s3.json \
+              -ip=127.0.0.1 \
+              > /tmp/weed-mini.log 2>&1 &
+          echo $! > /tmp/weed-mini.pid
+
+          for i in $(seq 1 30); do
+            if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
+              echo "weed mini is ready"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "weed mini failed to start within 30s"
+          tail -50 /tmp/weed-mini.log
+          exit 1
+
+      - name: Run SDK V2 routing tests
+        env:
+          S3_ENDPOINT: http://127.0.0.1:8333
+          AWS_ACCESS_KEY_ID: some_access_key1
+          AWS_SECRET_ACCESS_KEY: some_secret_key1
+          AWS_REGION: us-east-1
+        run: go test -v -timeout=5m ./test/s3/sdk_v2_routing/...
+
+      - name: Stop weed mini
+        if: always()
+        run: |
+          if [ -f /tmp/weed-mini.pid ]; then
+            kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
+          fi
+
+      - name: Show server log on failure
+        if: failure()
+        run: |
+          echo "=== weed mini log (last 200 lines) ==="
+          tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
+
+      - name: Archive log
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: s3-sdk-v2-routing-server-log
+          path: /tmp/weed-mini.log
+          retention-days: 3

.github/workflows/samba-integration.yml

@@ -0,0 +1,120 @@
+name: "Samba on FUSE Integration"
+
+on:
+  push:
+    branches: [ master, main ]
+    paths:
+      - 'weed/mount/**'
+      - 'weed/filer/**'
+      - 'weed/cluster/**'
+      - 'test/samba/**'
+      - '.github/workflows/samba-integration.yml'
+  pull_request:
+    branches: [ master, main ]
+    paths:
+      - 'weed/mount/**'
+      - 'weed/filer/**'
+      - 'weed/cluster/**'
+      - 'test/samba/**'
+      - '.github/workflows/samba-integration.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: samba-integration/${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  samba-integration:
+    name: samba-integration
+    runs-on: ubuntu-22.04
+    timeout-minutes: 45
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v6
+
+    - name: Set up Go
+      uses: actions/setup-go@v6
+      with:
+        go-version-file: 'go.mod'
+
+    - name: Start local Docker registry
+      run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v4
+      with:
+        driver-opts: network=host
+
+    - name: Build weed race binary
+      run: |
+        cd docker
+        make binary_race
+
+    - name: Build SeaweedFS e2e image
+      uses: docker/build-push-action@v7
+      with:
+        context: docker
+        file: docker/Dockerfile.e2e
+        tags: localhost:5000/chrislusf/seaweedfs:e2e
+        push: true
+        cache-from: type=gha,scope=samba-e2e
+        cache-to: type=gha,mode=max,scope=samba-e2e
+
+    - name: Tag e2e image for docker compose
+      run: |
+        docker pull localhost:5000/chrislusf/seaweedfs:e2e
+        docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
+
+    - name: Build samba image
+      uses: docker/build-push-action@v7
+      with:
+        context: test/samba
+        build-contexts: |
+          chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
+        tags: localhost:5000/chrislusf/seaweedfs:samba
+        push: true
+        cache-from: type=gha,scope=samba-harness
+        cache-to: type=gha,mode=max,scope=samba-harness
+
+    - name: Tag samba image for docker compose
+      run: |
+        docker pull localhost:5000/chrislusf/seaweedfs:samba
+        docker tag localhost:5000/chrislusf/seaweedfs:samba chrislusf/seaweedfs:samba
+
+    - name: Start SeaweedFS cluster and Samba
+      run: |
+        docker compose -f test/samba/docker-compose.yml up --wait
+
+    - name: Run Samba test battery
+      run: |
+        set -o pipefail
+        docker compose -f test/samba/docker-compose.yml exec -T samba \
+          /run_inside_container.sh 2>&1 | tee /tmp/samba-output.log
+
+    - name: Collect logs
+      if: always()
+      run: |
+        mkdir -p /tmp/samba-docker-logs
+        for svc in master volume filer samba; do
+          docker compose -f test/samba/docker-compose.yml logs "$svc" \
+            > "/tmp/samba-docker-logs/${svc}.log" 2>&1 || true
+        done
+
+    - name: Tear down
+      if: always()
+      run: |
+        docker compose -f test/samba/docker-compose.yml down -v
+
+    - name: Upload logs
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: samba-integration-results
+        path: |
+          /tmp/samba-output.log
+          /tmp/samba-docker-logs/
+        retention-days: 7

README.md

@@ -509,20 +509,22 @@ SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mong
 
 ### Compared to MinIO ###
 
-MinIO follows AWS S3 closely and is ideal for testing for S3 API. It has good UI, policies, versionings, etc. SeaweedFS is trying to catch up here. It is also possible to put MinIO as a gateway in front of SeaweedFS later.
+Please note, as Apr 25, 2026 MinIO ceased developement. It's strongly discouraged to use that unmaintained software with multiple security bugs.
 
-MinIO metadata are in simple files. Each file write will incur extra writes to corresponding meta file.
+MinIO followed AWS S3 closely and was ideal for testing for S3 API. It had good UI, policies, versionings, etc. SeaweedFS is trying to catch up here. 
 
-MinIO does not have optimization for lots of small files. The files are simply stored as is to local disks.
+MinIO metadata were in simple files. Each file write will incur extra writes to corresponding meta file.
+
+MinIO did not have optimization for lots of small files. The files were simply stored as is to local disks.
 Plus the extra meta file and shards for erasure coding, it only amplifies the LOSF problem.
 
-MinIO has multiple disk IO to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files.
+MinIO had multiple disk IO to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files.
 
-MinIO has full-time erasure coding. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data.
+MinIO had full-time erasure coding. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data.
 
-MinIO does not have POSIX-like API support.
+MinIO did not have POSIX-like API support.
 
-MinIO has specific requirements on storage layout. It is not flexible to adjust capacity. In SeaweedFS, just start one volume server pointing to the master. That's all.
+MinIO had specific requirements on storage layout. It is not flexible to adjust capacity. In SeaweedFS, just start one volume server pointing to the master. That's all.
 
 ## Dev Plan ##
 

design-bucket-config-serialization.md

@@ -0,0 +1,167 @@
+# Design: Serializing Bucket Configuration Mutations
+
+Issue #9651 — concurrent `PutBucketVersioning` + `PutBucketEncryption` (as Terraform
+issues them in parallel) intermittently lose the encryption write.
+
+## Root cause
+
+The bucket's entire config lives in one filer entry, `/buckets/<name>`. Every
+config API does a read-modify-write of that single entry, and the writes are not
+serialized:
+
+- `updateBucketConfig(bucket, fn)` (`s3api_bucket_config.go:468`) — sources from a
+  possibly-stale cached `BucketConfig`, mutates `Entry.Extended`, writes the
+  **whole** entry. Used by: versioning, object-lock config, lifecycle, ACL/owner.
+- `UpdateBucketMetadata` → `setBucketMetadata` (`:1042`) — reads a fresh entry,
+  mutates `Entry.Content`, writes the **whole** entry. Used by: encryption, CORS,
+  tagging, ownership, policy, notification.
+
+Two ingredients produce the lost update:
+
+1. **No serialization** of the read→modify→write (the cache mutexes only guard the
+   in-memory map, not the RMW).
+2. **Whole-entry rewrite from an independent snapshot** — `updateBucketConfig`
+   rebuilds from a stale cached `BucketConfig` whose `Content` predates the
+   concurrent encryption write, so writing the whole entry reverts `Content`.
+
+Sequential calls always pass (each sees the previous write), so it only surfaces
+under concurrency — and CI's slower IO widens the window (the "2 of ~12 runs").
+
+## Goals
+
+- No lost updates across concurrent bucket-config changes — for **all** config
+  fields, not just versioning/encryption.
+- Correct for a single S3 gateway (the reported case) and for multiple gateways.
+- Reuse the filer primitives just merged (per-path lock, `WriteCondition`,
+  `ObjectTransaction`); do not reintroduce a distributed lock.
+- Minimal blast radius: the fix lands at the two chokepoint helpers.
+
+## Non-goals
+
+- Changing the one-entry-per-bucket storage model.
+- Multi-filer-concurrent bucket writes (addressed only as an optional phase 3).
+
+## The two ingredients map to two complementary fixes
+
+### Fix A — serialize + read fresh (closes the window for whole-entry writers)
+
+Both `updateBucketConfig` and `UpdateBucketMetadata` must run their RMW under one
+per-bucket critical section, and **re-read the entry fresh from the filer inside
+it** — not rebuild from the cached `BucketConfig`. The lock alone is insufficient:
+without the fresh read, two serialized writers still each apply a stale snapshot.
+
+### Fix B — field-level updates (removes the collision entirely)
+
+The two writers touch disjoint fields (`Extended[versioning]` vs `Content`). If
+each path updated only its own field instead of rewriting the whole entry, neither
+could clobber the other regardless of ordering. This is the structural fix and
+makes serialization a defense-in-depth concern rather than a correctness
+requirement for cross-field cases.
+
+## Where to serialize (layering)
+
+The bucket entry is a single filer entry, so unlike object writes there is no
+sharding — the question is purely the scope of the lock:
+
+| Layer | Serializes across | Cost | Notes |
+|---|---|---|---|
+| 1. Gateway-local per-bucket lock | one gateway process | tiny | fixes the reported (single-gateway/CI) case |
+| 2. Filer per-path lock via conditional write | all gateways on one filer | small | reuses #9640 `CreateEntry`+`WriteCondition` |
+| 3. Route-by-key to bucket-key owner filer | all gateways and filers | medium | same mechanism as the object DLM-removal |
+
+## Recommended plan (phased)
+
+### Phase 1 — minimal fix for #9651 (gateway-local lock + fresh read)
+
+Add a bounded per-bucket lock table to `S3ApiServer`, reusing the same
+`util.LockTable` the filer uses for its per-path lock:
+
+```go
+// in S3ApiServer
+bucketConfigLocks *util.LockTable[string] // serialize bucket-entry RMW
+
+func (s3a *S3ApiServer) withBucketConfigLock(bucket string, fn func() s3err.ErrorCode) s3err.ErrorCode {
+    lk := s3a.bucketConfigLocks.AcquireLock("bucketConfig", bucket, util.ExclusiveLock)
+    defer s3a.bucketConfigLocks.ReleaseLock(bucket, lk)
+    return fn()
+}
+```
+
+Wrap the RMW in **both** chokepoints, and inside the lock read the entry fresh:
+
+- `updateBucketConfig`: acquire the lock; re-read `/buckets/<name>` from the filer
+  (not the cache); rebuild `BucketConfig` from that fresh entry; apply `fn`; write;
+  invalidate cache; release.
+- `UpdateBucketMetadata`/`setBucketMetadata`: same lock key; it already reads fresh,
+  so it just needs to share the critical section.
+
+Both must use the **same** lock keyed on `bucket`, so versioning and encryption
+contend on one mutex. This closes the reported window. Limitation: only one
+gateway; two gateways behind a load balancer still race.
+
+Test: parallel `PutBucketVersioning` + `PutBucketEncryption`, assert both persist
+(the exact Terraform scenario), plus an N-way parallel variant over distinct
+fields.
+
+### Phase 2 — robust across gateways (field-level + CAS via merged primitives)
+
+Move the writers off whole-entry rewrites:
+
+- **Extended-based config** (versioning, object-lock, ownership, tagging-in-Extended)
+  → `ObjectTransaction` `PATCH_EXTENDED` on `/buckets/<name>`. The owner filer reads
+  the entry fresh under its per-path lock and merges only the named keys, so the
+  gateway never sends a whole-entry snapshot — this dissolves *both* ingredients for
+  these fields.
+- **`Content`-based config** (encryption, CORS, tags blob) — **chosen and
+  implemented (b3): extend `PATCH_EXTENDED` with `set_content`.** Under the same
+  per-path lock the filer reads the entry fresh, merges extended attributes, and
+  replaces `Content`, preserving the rest. So a content write becomes a field-level
+  patch too — `setBucketMetadata` patches `Content`, `updateBucketConfig` patches
+  extended keys, and the two serialize on the lock instead of racing whole-entry
+  rewrites. This is cleaner than the alternatives below: no client-side retry, no
+  storage migration, and it reuses `ObjectTransaction`'s existing atomic lock.
+  - (b1, rejected) Conditional `CreateEntry` overwrite with `IF_ETAG_MATCH` + retry
+    (#9640): correct but needs client-side retry, and the bucket directory entry has
+    no reliable ETag to compare on.
+  - (b2, future) Migrate each per-feature config out of the single `Content` blob
+    into its own `Extended` key. Then even *intra-blob* writes (tags vs encryption)
+    stop racing. Larger migration; tracked separately.
+
+Once all paths are field-level patches, the phase-1 gateway lock is unnecessary —
+the filer enforces atomicity. (This is the path taken: phase 1 was skipped.)
+
+### Phase 3 — multi-filer (only if needed)
+
+If multiple filers can write `/buckets/<name>` concurrently, a filer-local per-path
+lock no longer suffices. Route bucket-config writes to
+`PrimaryForKey("/buckets/<name>")` (the lock-ring view) and serialize on that one
+owner filer — the same route-by-key design used to take object writes off the DLM.
+Overkill for rare config writes; include only if multi-filer bucket writes are real.
+
+## Correctness summary
+
+- Phase 1: all RMW for a bucket serialize within a gateway; the fresh read means the
+  second writer observes the first's change. Closes #9651 for single-gateway.
+- Phase 2: `PATCH_EXTENDED` is atomic field-level merge at the filer (no snapshot);
+  CAS turns a concurrent `Content` write into a retry, enforced under the filer's
+  per-path lock — correct for any number of gateways sharing a filer.
+- Phase 3: one owner filer serializes all writers — correct across filers too.
+
+## Scope checklist (every path that RMWs the bucket entry)
+
+All of these funnel through the two chokepoints, so fixing the chokepoints covers
+them — but the fix must not leave any of them on an unserialized path:
+
+- via `updateBucketConfig`: versioning, object-lock config, lifecycle, ACL/owner.
+- via `UpdateBucketMetadata`/`setBucketMetadata`: encryption, CORS, tagging,
+  ownership controls, bucket policy, notification.
+- bucket create/delete (`CreateEntry`/`DeleteEntry` of `/buckets/<name>`) already
+  go through the filer's per-path lock on `CreateEntry`; ensure they take the same
+  bucket lock if they also patch config.
+
+## Cache rule (must document in code)
+
+Under the lock, **read the entry from the filer, never rebuild from the cached
+`BucketConfig`**. The cache is for reads; it must be invalidated on every write and
+never be the source for an RMW. This is the single most important detail — the lock
+without the fresh read does not fix the bug.

docker/Dockerfile.go_build

@@ -42,6 +42,10 @@ RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
       echo "Skipping Rust build for $TARGETARCH (unsupported)" && \
       touch /weed-volume; \
     fi
+# Pre-built binaries arrive via GitHub Actions artifacts, which drop the
+# executable bit, so the copied file is 0644 and exec fails with "Permission
+# denied". Restore it (no-op for the empty placeholder, which stays size 0).
+RUN chmod 0755 /weed-volume
 
 FROM alpine AS final
 LABEL author="Chris Lu"

go.mod

@@ -15,7 +15,6 @@ require (
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.6.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dustin/go-humanize v1.0.1
 	github.com/eapache/go-resiliency v1.6.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
@@ -27,7 +26,7 @@ require (
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/go-redsync/redsync/v4 v4.16.0
-	github.com/go-sql-driver/mysql v1.9.3
+	github.com/go-sql-driver/mysql v1.10.0
 	github.com/go-zookeeper/zk v1.0.4 // indirect
 	github.com/golang/protobuf v1.5.4
 	github.com/golang/snappy v1.0.0
@@ -49,7 +48,7 @@ require (
 	github.com/klauspost/compress v1.18.6
 	github.com/klauspost/reedsolomon v1.14.0
 	github.com/kurin/blazer v0.5.3
-	github.com/linxGnu/grocksdb v1.10.7
+	github.com/linxGnu/grocksdb v1.10.8
 	github.com/mailru/easyjson v0.9.1 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -92,13 +91,13 @@ require (
 	gocloud.dev v0.45.0
 	gocloud.dev/pubsub/natspubsub v0.45.0
 	gocloud.dev/pubsub/rabbitpubsub v0.45.0
-	golang.org/x/crypto v0.50.0
+	golang.org/x/crypto v0.52.0
 	golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
 	golang.org/x/image v0.39.0
-	golang.org/x/net v0.53.0
+	golang.org/x/net v0.54.0
 	golang.org/x/oauth2 v0.36.0
-	golang.org/x/sys v0.43.0
-	golang.org/x/text v0.36.0 // indirect
+	golang.org/x/sys v0.45.0
+	golang.org/x/text v0.37.0 // indirect
 	golang.org/x/tools v0.44.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.278.0
@@ -123,11 +122,11 @@ require (
 	github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
 	github.com/arangodb/go-driver v1.6.9
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.41.6
+	github.com/aws/aws-sdk-go-v2 v1.41.7
 	github.com/aws/aws-sdk-go-v2/config v1.32.14
 	github.com/aws/aws-sdk-go-v2/credentials v1.19.14
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0
-	github.com/cognusion/imaging v1.0.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
+	github.com/cognusion/imaging v1.0.3
 	github.com/fluent/fluent-logger-golang v1.10.1
 	github.com/getsentry/sentry-go v0.44.1
 	github.com/go-git/go-billy/v5 v5.9.0
@@ -141,12 +140,12 @@ require (
 	github.com/linkedin/goavro/v2 v2.15.0
 	github.com/minio/crc64nvme v1.1.1
 	github.com/orcaman/concurrent-map/v2 v2.0.1
-	github.com/parquet-go/parquet-go v0.28.0
+	github.com/parquet-go/parquet-go v0.30.1
 	github.com/pkg/sftp v1.13.10
 	github.com/rabbitmq/amqp091-go v1.11.0
 	github.com/rclone/rclone v1.74.1
 	github.com/rdleal/intervalst v1.5.0
-	github.com/redis/go-redis/v9 v9.18.0
+	github.com/redis/go-redis/v9 v9.19.0
 	github.com/schollz/progressbar/v3 v3.19.0
 	github.com/seaweedfs/go-fuse/v2 v2.9.3
 	github.com/shirou/gopsutil/v4 v4.26.3
@@ -158,7 +157,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
 	github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2
-	go.etcd.io/etcd/client/pkg/v3 v3.6.10
+	go.etcd.io/etcd/client/pkg/v3 v3.6.11
 	go.uber.org/atomic v1.11.0
 	golang.org/x/sync v0.20.0
 	golang.org/x/tools/godoc v0.1.0-deprecated
@@ -297,14 +296,14 @@ require (
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	cloud.google.com/go/iam v1.7.0 // indirect
 	cloud.google.com/go/monitoring v1.24.3 // indirect
-	filippo.io/edwards25519 v1.1.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
+	filippo.io/edwards25519 v1.2.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 // indirect
 	github.com/Azure/go-ntlmssp v0.1.1 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
 	github.com/Files-com/files-sdk-go/v3 v3.3.82 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
@@ -324,17 +323,17 @@ require (
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
@@ -499,7 +498,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.43.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
-	golang.org/x/term v0.42.0
+	golang.org/x/term v0.43.0
 	golang.org/x/time v0.15.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect

go.sum

@@ -547,28 +547,28 @@ cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcP
 dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
 dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw=
-filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
+filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
 gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8=
 git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 h1:jHb/wfvRikGdxMXYV3QG/SzUOPYN9KEUUuC0Yd0/vC0=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1/go.mod h1:pzBXCYn05zvYIrwLgtK8Ap8QcjRg+0i76tMQdWN6wOk=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY=
 github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 h1:m/sWOGCREuSBqg2htVQTBY8nOZpyajYztF0vUvSZTuM=
 github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0/go.mod h1:Pu5Zksi2KrU7LPbZbNINx6fuVrUp/ffvpxdDj+i8LeE=
 github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw=
 github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 h1:BM85pSYlVYQHdq00nxyPoOkyLF5NArJG3bOsrmbwr4k=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0/go.mod h1:QYjP2cB7ZYtS/8jAbE0VSBZde/tjExqGjp+8JY6/+ts=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 h1:tZh20RjgfMxKBxJiIS75iTVAKIUxrST5X2dVHMTptL4=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4/go.mod h1:vGYAk36rhMVCfTP7v+RVruCR0zmPe6S+36KRpDCLySw=
 github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
@@ -581,8 +581,8 @@ github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDP
 github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d h1:iG9B49Q218F/XxXNRM7k/vWf7MKmLIS8AcJV9cGN4nA=
@@ -715,10 +715,10 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.41.6 h1:1AX0AthnBQzMx1vbmir3Y4WsnJgiydmnJjiLu+LvXOg=
-github.com/aws/aws-sdk-go-v2 v1.41.6/go.mod h1:dy0UzBIfwSeot4grGvY1AqFWN5zgziMmWGzysDnHFcQ=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
+github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
+github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
 github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI=
 github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo=
 github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
@@ -727,24 +727,24 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeD
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 h1:uMC4oL6G3MNhodo358QEqSDjrgvzV3TUQ58nyQSGq2E=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13/go.mod h1:Cer86AE2686DvVUe57LPve3jUBmbujuaonSX8pNzGgw=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
 github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
@@ -866,8 +866,8 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/version v0.0.0-20250314144055-3860cd14adf2 h1:8Vfw2iNEpYIV6aLtMwT5UOGuPmp9MKlEKWKFTuB+MPU=
 github.com/cockroachdb/version v0.0.0-20250314144055-3860cd14adf2/go.mod h1:P9WiZOdQ1R/ZZDL0WzF5wlyRvrjtfhNOwMZymFpBwjE=
-github.com/cognusion/imaging v1.0.2 h1:BQwBV8V8eF3+dwffp8Udl9xF1JKh5Z0z5JkJwAi98Mc=
-github.com/cognusion/imaging v1.0.2/go.mod h1:mj7FvH7cT2dlFogQOSUQRtotBxJ4gFQ2ySMSmBm5dSk=
+github.com/cognusion/imaging v1.0.3 h1:nHyIeEVDV8JkBbuhgx8iSBW72W8rHbDEyruA0Jh7Lnk=
+github.com/cognusion/imaging v1.0.3/go.mod h1:38tFLFhGK81ORThZG8dVXPtp1uhd+3xM1WXuMn+unOA=
 github.com/colinmarc/hdfs/v2 v2.4.0 h1:v6R8oBx/Wu9fHpdPoJJjpGSUxo8NhHIwrwsfhFvU9W0=
 github.com/colinmarc/hdfs/v2 v2.4.0/go.mod h1:0NAO+/3knbMx6+5pCv+Hcbaz4xn/Zzbn9+WIib2rKVI=
 github.com/compose-spec/compose-go/v2 v2.9.0 h1:UHSv/QHlo6QJtrT4igF1rdORgIUhDo1gWuyJUoiNNIM=
@@ -1128,8 +1128,8 @@ github.com/go-redsync/redsync/v4 v4.16.0 h1:bNcOzeHH9d3s6pghU9NJFMPrQa41f5Nx3L4Y
 github.com/go-redsync/redsync/v4 v4.16.0/go.mod h1:V4gagqgyASWBZuwx4xGzu72aZNb/6Mo05byUa3mVmKQ=
 github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk=
 github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
-github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
-github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
+github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
+github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
@@ -1515,8 +1515,8 @@ github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI=
 github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
 github.com/linkedin/goavro/v2 v2.15.0 h1:pDj1UrjUOO62iXhgBiE7jQkpNIc5/tA5eZsgolMjgVI=
 github.com/linkedin/goavro/v2 v2.15.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
-github.com/linxGnu/grocksdb v1.10.7 h1:fCi4qvZWo04VgFwGWmO8HQJgUVounJBy+C2TMVPU/ho=
-github.com/linxGnu/grocksdb v1.10.7/go.mod h1:OLQKZwiKwaJiAVCsOzWKvwiLwfZ5Vz8Md5TYR7t7pM8=
+github.com/linxGnu/grocksdb v1.10.8 h1:Nau01Hhm/0kaVTR6d4viwD6npYbnDvZAfzwJCLzKRYo=
+github.com/linxGnu/grocksdb v1.10.8/go.mod h1:OLQKZwiKwaJiAVCsOzWKvwiLwfZ5Vz8Md5TYR7t7pM8=
 github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=
 github.com/lithammer/fuzzysearch v1.1.8/go.mod h1:IdqeyBClc3FFqSzYq/MXESsS4S0FsZ5ajtkr5xPLts4=
 github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8=
@@ -1667,8 +1667,8 @@ github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxP
 github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
 github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU=
 github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
-github.com/parquet-go/parquet-go v0.28.0 h1:ECyksyv8T2pOrlLsN7aWJIoQakyk/HtxQ2lchgS4els=
-github.com/parquet-go/parquet-go v0.28.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
+github.com/parquet-go/parquet-go v0.30.1 h1:Oy6ganNrAdFiVwy7wNmWagfPTWA2X9Z3tVHBc7JtuX8=
+github.com/parquet-go/parquet-go v0.30.1/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
@@ -1795,8 +1795,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
 github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
-github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
-github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
+github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k=
+github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
 github.com/redis/rueidis v1.0.71 h1:pODtnAR5GAB7j4ekhldZ29HKOxe4Hph0GTDGk1ayEQY=
 github.com/redis/rueidis v1.0.71/go.mod h1:lfdcZzJ1oKGKL37vh9fO3ymwt+0TdjkkUCJxbgpmcgQ=
 github.com/redis/rueidis/rueidiscompat v1.0.71 h1:wNZ//kEjMZgBM0KCk7ncOX8KmAgROU2kDdDNpwheG4w=
@@ -2111,8 +2111,8 @@ go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
 go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
 go.etcd.io/etcd/api/v3 v3.6.10 h1:jlwjtELjA8yi2VWpOFH+0w0lGr3K6mVDyn0RDB9aaAY=
 go.etcd.io/etcd/api/v3 v3.6.10/go.mod h1:pdV4VeFmvhdNjB4LWRkC8ReLyRBAxUOze3GarMhE2sk=
-go.etcd.io/etcd/client/pkg/v3 v3.6.10 h1:tBT7podcPhuVbCVkAEzx8bC5I+aqxfLwBN8/As1arrA=
-go.etcd.io/etcd/client/pkg/v3 v3.6.10/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
+go.etcd.io/etcd/client/pkg/v3 v3.6.11 h1:e41mp315Yn3QMGPmEzCyLsMINgJXTY/dX8kM++1csxU=
+go.etcd.io/etcd/client/pkg/v3 v3.6.11/go.mod h1:DysuMe/inqRyC/1tjRR6hReH/VV9Lufs27YKSKBWWJg=
 go.etcd.io/etcd/client/v3 v3.6.10 h1:J598zJ+C/ZPvImypmq5waj84+bovePrlZERHklf34y0=
 go.etcd.io/etcd/client/v3 v3.6.10/go.mod h1:iHhUDUcEwaKs1YFq3MgmI9U4zhTVasp/vgdVbFf1RS8=
 go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
@@ -2216,8 +2216,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
-golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
+golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
+golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2352,8 +2352,8 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
-golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
+golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
+golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2511,8 +2511,8 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
-golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
+golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4=
 golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE=
@@ -2531,8 +2531,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
-golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
-golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
+golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
+golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2553,8 +2553,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
-golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
+golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
+golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.25"
+appVersion: "4.29"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.25.0
+version: 4.29.0

k8s/charts/seaweedfs/templates/admin/admin-statefulset.yaml

@@ -135,6 +135,14 @@ spec:
                 {{ toYaml $value | nindent 16 | trim }}
             {{- end -}}
             {{- end }}
+            {{- $secretExtraEnvironmentVars := .Values.admin.secretExtraEnvironmentVars }}
+            {{- if $secretExtraEnvironmentVars }}
+            {{- range $key := keys $secretExtraEnvironmentVars | sortAlpha }}
+            {{- $value := index $secretExtraEnvironmentVars $key }}
+            - name: {{ $key }}
+              valueFrom: {{ toYaml $value | nindent 16 }}
+            {{- end }}
+            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"
@@ -180,11 +188,13 @@ spec:
             - name: admin-logs
               mountPath: /logs
             {{- end }}
-            {{- if .Values.global.seaweedfs.enableSecurity }}
+            {{- if include "seaweedfs.securityConfigEnabled" . }}
             - name: security-config
               readOnly: true
               mountPath: /etc/seaweedfs/security.toml
               subPath: security.toml
+            {{- end }}
+            {{- if .Values.global.seaweedfs.enableSecurity }}
             - name: ca-cert
               readOnly: true
               mountPath: /usr/local/share/ca-certificates/ca/
@@ -275,10 +285,12 @@ spec:
           persistentVolumeClaim:
             claimName: {{ .Values.admin.logs.claimName }}
         {{- end }}
-        {{- if .Values.global.seaweedfs.enableSecurity }}
+        {{- if include "seaweedfs.securityConfigEnabled" . }}
         - name: security-config
           configMap:
             name: {{ include "seaweedfs.fullname" . }}-security-config
+        {{- end }}
+        {{- if .Values.global.seaweedfs.enableSecurity }}
         - name: ca-cert
           secret:
             secretName: {{ include "seaweedfs.fullname" . }}-ca-cert

k8s/charts/seaweedfs/templates/all-in-one/all-in-one-deployment.yaml

@@ -330,11 +330,13 @@ spec:
               mountPath: /etc/seaweedfs/master.toml
               subPath: master.toml
               readOnly: true
-            {{- if .Values.global.seaweedfs.enableSecurity }}
+            {{- if include "seaweedfs.securityConfigEnabled" . }}
             - name: security-config
               mountPath: /etc/seaweedfs/security.toml
               subPath: security.toml
               readOnly: true
+            {{- end }}
+            {{- if .Values.global.seaweedfs.enableSecurity }}
             - name: ca-cert
               mountPath: /usr/local/share/ca-certificates/ca/
               readOnly: true
@@ -461,10 +463,12 @@ spec:
         - name: master-config
           configMap:
             name: {{ include "seaweedfs.fullname" . }}-master-config
-        {{- if .Values.global.seaweedfs.enableSecurity }}
+        {{- if include "seaweedfs.securityConfigEnabled" . }}
         - name: security-config
           configMap:
             name: {{ include "seaweedfs.fullname" . }}-security-config
+        {{- end }}
+        {{- if .Values.global.seaweedfs.enableSecurity }}
         - name: ca-cert
           secret:
             secretName: {{ include "seaweedfs.fullname" . }}-ca-cert

k8s/charts/seaweedfs/templates/cosi/cosi-deployment.yaml

@@ -117,11 +117,13 @@ spec:
               name: config-users
               readOnly: true
             {{- end }}
-            {{- if .Values.global.seaweedfs.enableSecurity }}
+            {{- if include "seaweedfs.securityConfigEnabled" . }}
             - name: security-config
               readOnly: true
               mountPath: /etc/seaweedfs/security.toml
               subPath: security.toml
+            {{- end }}
+            {{- if .Values.global.seaweedfs.enableSecurity }}
             - name: ca-cert
               readOnly: true
               mountPath: /usr/local/share/ca-certificates/ca/
@@ -179,10 +181,12 @@ spec:
             secretName: {{ include "seaweedfs.fullname" . }}-s3-secret
             {{- end }}
         {{- end }}
-        {{- if .Values.global.seaweedfs.enableSecurity }}
+        {{- if include "seaweedfs.securityConfigEnabled" . }}
         - name: security-config
           configMap:
             name: {{ include "seaweedfs.fullname" . }}-security-config
+        {{- end }}
+        {{- if .Values.global.seaweedfs.enableSecurity }}
         - name: ca-cert
           secret:
             secretName: {{ include "seaweedfs.fullname" . }}-ca-cert

k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml

@@ -234,11 +234,13 @@ spec:
               mountPath: /etc/seaweedfs/notification.toml
               subPath: notification.toml
             {{- end }}
-            {{- if .Values.global.seaweedfs.enableSecurity }}
+            {{- if include "seaweedfs.securityConfigEnabled" . }}
             - name: security-config
               readOnly: true
               mountPath: /etc/seaweedfs/security.toml
               subPath: security.toml
+            {{- end }}
+            {{- if .Values.global.seaweedfs.enableSecurity }}
             - name: ca-cert
               readOnly: true
               mountPath: /usr/local/share/ca-certificates/ca/
@@ -274,7 +276,8 @@ spec:
               name: swfs-s3-tls
             {{- end }}
             {{- end }}
-          {{- $isJwtEnabled := or .Values.global.seaweedfs.securityConfig.jwtSigning.filerWrite .Values.global.seaweedfs.securityConfig.jwtSigning.filerRead }}
+          {{- $jwt := (.Values.global.seaweedfs.securityConfig).jwtSigning | default dict }}
+          {{- $isJwtEnabled := or $jwt.filerWrite $jwt.filerRead }}
           {{- if .Values.filer.readinessProbe.enabled }}
           readinessProbe:
           {{- if or $isJwtEnabled .Values.filer.readinessProbe.tcpSocket }}
@@ -368,10 +371,12 @@ spec:
           configMap:
             name: {{ include "seaweedfs.fullname" . }}-notification-config
         {{- end }}
-        {{- if .Values.global.seaweedfs.enableSecurity }}
+        {{- if include "seaweedfs.securityConfigEnabled" . }}
         - name: security-config
           configMap:
             name: {{ include "seaweedfs.fullname" . }}-security-config
+        {{- end }}
+        {{- if .Values.global.seaweedfs.enableSecurity }}
         - name: ca-cert
           secret:
             secretName: {{ include "seaweedfs.fullname" . }}-ca-cert

k8s/charts/seaweedfs/templates/master/master-statefulset.yaml

@@ -188,11 +188,13 @@ spec:
               readOnly: true
               mountPath: /etc/seaweedfs/master.toml
               subPath: master.toml
-            {{- if .Values.global.seaweedfs.enableSecurity }}
+            {{- if include "seaweedfs.securityConfigEnabled" . }}
             - name: security-config
               readOnly: true
               mountPath: /etc/seaweedfs/security.toml
               subPath: security.toml
+            {{- end }}
+            {{- if .Values.global.seaweedfs.enableSecurity }}
             - name: ca-cert
               readOnly: true
               mountPath: /usr/local/share/ca-certificates/ca/
@@ -287,10 +289,12 @@ spec:
         - name: master-config
           configMap:
             name: {{ include "seaweedfs.fullname" . }}-master-config
-        {{- if .Values.global.seaweedfs.enableSecurity }}
+        {{- if include "seaweedfs.securityConfigEnabled" . }}
         - name: security-config
           configMap:
             name: {{ include "seaweedfs.fullname" . }}-security-config
+        {{- end }}
+        {{- if .Values.global.seaweedfs.enableSecurity }}
         - name: ca-cert
           secret:
             secretName: {{ include "seaweedfs.fullname" . }}-ca-cert

k8s/charts/seaweedfs/templates/s3/s3-deployment.yaml

@@ -156,11 +156,13 @@ spec:
               name: config-users
               readOnly: true
             {{- end }}
-            {{- if .Values.global.seaweedfs.enableSecurity }}
+            {{- if include "seaweedfs.securityConfigEnabled" . }}
             - name: security-config
               readOnly: true
               mountPath: /etc/seaweedfs/security.toml
               subPath: security.toml
+            {{- end }}
+            {{- if .Values.global.seaweedfs.enableSecurity }}
             - name: ca-cert
               readOnly: true
               mountPath: /usr/local/share/ca-certificates/ca/
@@ -249,10 +251,12 @@ spec:
         - name: logs
           emptyDir: {}
         {{- end }}
-        {{- if .Values.global.seaweedfs.enableSecurity }}
+        {{- if include "seaweedfs.securityConfigEnabled" . }}
         - name: security-config
           configMap:
             name: {{ include "seaweedfs.fullname" . }}-security-config
+        {{- end }}
+        {{- if .Values.global.seaweedfs.enableSecurity }}
         - name: ca-cert
           secret:
             secretName: {{ include "seaweedfs.fullname" . }}-ca-cert

k8s/charts/seaweedfs/templates/sftp/sftp-deployment.yaml

@@ -176,11 +176,13 @@ spec:
             - mountPath: /etc/sw/ssh
               name: config-ssh
               readOnly: true
-            {{- if .Values.global.seaweedfs.enableSecurity }}
+            {{- if include "seaweedfs.securityConfigEnabled" . }}
             - name: security-config
               readOnly: true
               mountPath: /etc/seaweedfs/security.toml
               subPath: security.toml
+            {{- end }}
+            {{- if .Values.global.seaweedfs.enableSecurity }}
             - name: ca-cert
               readOnly: true
               mountPath: /usr/local/share/ca-certificates/ca/
@@ -264,10 +266,12 @@ spec:
         - name: logs
           emptyDir: {}
         {{- end }}
-        {{- if .Values.global.seaweedfs.enableSecurity }}
+        {{- if include "seaweedfs.securityConfigEnabled" . }}
         - name: security-config
           configMap:
             name: {{ include "seaweedfs.fullname" . }}-security-config
+        {{- end }}
+        {{- if .Values.global.seaweedfs.enableSecurity }}
         - name: ca-cert
           secret:
             secretName: {{ include "seaweedfs.fullname" . }}-ca-cert

k8s/charts/seaweedfs/templates/shared/_helpers.tpl

@@ -332,6 +332,16 @@ Create the name of the service account to use
 {{- .Values.global.seaweedfs.serviceAccountName | default "seaweedfs" -}}
 {{- end -}}
 
+{{/* True when security.toml should be rendered and mounted. volumeWrite is
+     excluded since it defaults to true. */}}
+{{- define "seaweedfs.securityConfigEnabled" -}}
+{{- $sec := (.Values.global.seaweedfs).securityConfig | default dict -}}
+{{- $jwt := $sec.jwtSigning | default dict -}}
+{{- if or .Values.global.seaweedfs.enableSecurity $jwt.volumeRead $jwt.filerWrite $jwt.filerRead -}}
+true
+{{- end -}}
+{{- end -}}
+
 {{/* S3 TLS cert/key arguments, using custom secret if s3.tlsSecret is set */}}
 {{- define "seaweedfs.s3.tlsArgs" -}}
 {{- $prefix := .prefix -}}

k8s/charts/seaweedfs/templates/shared/security-configmap.yaml

@@ -1,5 +1,5 @@
 {{- include "seaweedfs.compat" . -}}
-{{- if .Values.global.seaweedfs.enableSecurity }}
+{{- if include "seaweedfs.securityConfigEnabled" . }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -55,6 +55,7 @@ data:
     key = "{{ dig "jwt" "filer_signing" "read" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
     {{- end }}
 
+    {{- if .Values.global.seaweedfs.enableSecurity }}
     # all grpc tls authentications are mutual
     # the values for the following ca, cert, and key are paths to the PERM files.
     [grpc]
@@ -94,4 +95,5 @@ data:
     [https.volume]
     cert = ""
     key  = ""
+    {{- end }}
 {{- end }}

k8s/charts/seaweedfs/templates/volume/volume-statefulset.yaml

@@ -137,6 +137,9 @@ spec:
             - "/bin/sh"
             - "-ec"
             - |
+              {{- if $volume.rust }}
+              exec /usr/bin/weed-volume \
+              {{- else }}
               exec /usr/bin/weed \
                 {{- if $volume.logs }}
                 -logdir=/logs \
@@ -149,6 +152,7 @@ spec:
                 -v={{ $.Values.global.seaweedfs.loggingLevel }} \
                 {{- end }}
                 volume \
+              {{- end }}
                 -port={{ $volume.port }} \
                 {{- if $volume.metricsPort }}
                 -metricsPort={{ $volume.metricsPort }} \
@@ -180,7 +184,7 @@ spec:
                 {{- if $volume.imagesFixOrientation }}
                 -images.fix.orientation \
                 {{- end }}
-                {{- if $volume.pulseSeconds }}
+                {{- if and $volume.pulseSeconds (not $volume.rust) }}
                 -pulseSeconds={{ $volume.pulseSeconds }} \
                 {{- end }}
                 {{- if $volume.index }}
@@ -211,11 +215,13 @@ spec:
             - name: idx
               mountPath: "/idx/"
             {{- end }}
-            {{- if $.Values.global.seaweedfs.enableSecurity }}
+            {{- if include "seaweedfs.securityConfigEnabled" $ }}
             - name: security-config
               readOnly: true
               mountPath: /etc/seaweedfs/security.toml
               subPath: security.toml
+            {{- end }}
+            {{- if $.Values.global.seaweedfs.enableSecurity }}
             - name: ca-cert
               readOnly: true
               mountPath: /usr/local/share/ca-certificates/ca/
@@ -333,10 +339,12 @@ spec:
           emptyDir: {}
        {{- end }}
      {{- end }}
-     {{- if $.Values.global.seaweedfs.enableSecurity }}
+     {{- if include "seaweedfs.securityConfigEnabled" $ }}
         - name: security-config
           configMap:
             name: {{ include "seaweedfs.fullname" $ }}-security-config
+     {{- end }}
+     {{- if $.Values.global.seaweedfs.enableSecurity }}
         - name: ca-cert
           secret:
             secretName: {{ include "seaweedfs.fullname" $ }}-ca-cert

k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml

@@ -149,11 +149,13 @@ spec:
             - name: worker-logs
               mountPath: /logs
             {{- end }}
-            {{- if .Values.global.seaweedfs.enableSecurity }}
+            {{- if include "seaweedfs.securityConfigEnabled" . }}
             - name: security-config
               readOnly: true
               mountPath: /etc/seaweedfs/security.toml
               subPath: security.toml
+            {{- end }}
+            {{- if .Values.global.seaweedfs.enableSecurity }}
             - name: ca-cert
               readOnly: true
               mountPath: /usr/local/share/ca-certificates/ca/
@@ -252,10 +254,12 @@ spec:
           persistentVolumeClaim:
             claimName: {{ .Values.worker.logs.claimName }}
         {{- end }}
-        {{- if .Values.global.seaweedfs.enableSecurity }}
+        {{- if include "seaweedfs.securityConfigEnabled" . }}
         - name: security-config
           configMap:
             name: {{ include "seaweedfs.fullname" . }}-security-config
+        {{- end }}
+        {{- if .Values.global.seaweedfs.enableSecurity }}
         - name: ca-cert
           secret:
             secretName: {{ include "seaweedfs.fullname" . }}-ca-cert

k8s/charts/seaweedfs/values.yaml

@@ -18,6 +18,8 @@ global:
     loggingLevel: 1
     enableSecurity: false
     masterServer: null
+    # filerWrite: true mounts security.toml on filer + admin without needing
+    # enableSecurity (mTLS); required for the Admin UI Users tab.
     securityConfig:
       jwtSigning:
         volumeWrite: true
@@ -303,6 +305,11 @@ volume:
   enabled: true
   imageOverride: null
   restartPolicy: null
+  # Run the Rust volume server (/usr/bin/weed-volume) instead of the Go one.
+  # Requires an image that ships the Rust binary (amd64/arm64). The Go-only
+  # log flags (-logtostderr/-logdir/-v) and -pulseSeconds are dropped; set log
+  # level via the RUST_LOG env var in extraEnvironmentVars if needed.
+  rust: false
   port: 8080
   grpcPort: 18080
   metricsPort: 9327
@@ -1286,6 +1293,13 @@ admin:
 
   extraEnvironmentVars: {}
 
+  # secret env variables (e.g. for injecting OIDC client secret from a Kubernetes Secret)
+  secretExtraEnvironmentVars: {}
+    # WEED_ADMIN_OIDC_CLIENT_SECRET:
+    #   secretKeyRef:
+    #     name: seaweedfs-admin-oidc
+    #     key: client_secret
+
   # Health checks
   livenessProbe:
     enabled: true

other/java/client/src/main/proto/filer.proto

@@ -22,12 +22,24 @@ service SeaweedFiler {
     rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) {
     }
 
+    rpc TouchAccessTime (TouchAccessTimeRequest) returns (TouchAccessTimeResponse) {
+    }
+
     rpc AppendToEntry (AppendToEntryRequest) returns (AppendToEntryResponse) {
     }
 
     rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
     }
 
+    rpc ObjectTransaction (ObjectTransactionRequest) returns (ObjectTransactionResponse) {
+    }
+
+    rpc ObjectTransactionBatch (ObjectTransactionBatchRequest) returns (ObjectTransactionBatchResponse) {
+    }
+
+    rpc PosixLock (PosixLockRequest) returns (PosixLockResponse) {
+    }
+
     rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
     }
     rpc StreamRenameEntry (StreamRenameEntryRequest) returns (stream StreamRenameEntryResponse) {
@@ -208,6 +220,8 @@ message FuseAttributes {
     int32 mtime_ns = 19; // nanosecond component of mtime (0-999999999)
     int32 ctime_ns = 20; // nanosecond component of ctime (0-999999999)
     int32 crtime_ns = 21; // nanosecond component of crtime (0-999999999)
+    int64 atime = 22; // unix time in seconds, last access time
+    int32 atime_ns = 23; // nanosecond component of atime (0-999999999)
 }
 
 message CreateEntryRequest {
@@ -217,6 +231,56 @@ message CreateEntryRequest {
     bool is_from_other_cluster = 4;
     repeated int32 signatures = 5;
     bool skip_check_parent_directory = 6;
+    // Optional precondition evaluated against the current entry atomically with
+    // the write, under the filer's per-path lock. The caller must route the
+    // key's writes to this entry's owner filer for the check to be authoritative.
+    WriteCondition condition = 7;
+}
+
+// WriteCondition is the precondition the filer evaluates against the existing
+// entry before writing, under the per-path lock. A failed condition returns
+// FilerError PRECONDITION_FAILED. The client maps request semantics (e.g. RFC
+// 7232) to clauses; the filer just compares.
+//
+// A condition is a list of clauses that ALL must hold (logical AND). One clause
+// is the common case; several express what a single comparison cannot: an ETag
+// set (If-Match / If-None-Match with multiple values), weak-ETag comparison, and
+// compound conditions (e.g. If-Match + If-Unmodified-Since together).
+message WriteCondition {
+    enum Kind {
+        NONE                  = 0; // unconditional
+        IF_NOT_EXISTS         = 1; // fail if the entry exists (If-None-Match: *)
+        IF_EXISTS             = 2; // fail if the entry is absent (If-Match: *)
+        IF_ETAG_MATCH         = 3; // fail if absent or etag matches none of the set (If-Match)
+        IF_ETAG_NOT_MATCH     = 4; // fail if present and etag matches any of the set (If-None-Match)
+        IF_UNMODIFIED_SINCE   = 5; // fail if present and mtime > unix_time
+        IF_MODIFIED_SINCE     = 6; // fail if present and mtime <= unix_time
+        IF_EXTENDED_NOT_EQUAL = 7; // fail if present and extended[ext_key] == ext_value
+        IF_EXTENDED_TIME_ELAPSED = 8; // fail if present and extended[ext_key] (unix seconds) is in the future
+    }
+    // Clause is one primitive comparison. IF_ETAG_MATCH holds when the current
+    // entry's ETag equals any value in etags; IF_ETAG_NOT_MATCH holds when it
+    // equals none. allow_weak permits weak-comparison (ignoring the W/ prefix).
+    //
+    // The IF_EXTENDED_* kinds are generic guards on an extended attribute, used
+    // to enforce object-lock without teaching the filer S3 semantics:
+    // IF_EXTENDED_NOT_EQUAL expresses a legal hold (block while a key equals a
+    // value), and IF_EXTENDED_TIME_ELAPSED expresses retention (block while a
+    // stored unix-second deadline is in the future, compared to the filer's
+    // clock). The caller composes these and, for governance-bypass, simply omits
+    // the retention clause when the bypass is authorized — the filer makes no
+    // authorization decision.
+    message Clause {
+        Kind kind = 1;
+        repeated string etags = 2; // ETag set for IF_ETAG_* kinds
+        int64 unix_time = 3;       // bound (unix seconds) for IF_*_SINCE kinds
+        bool allow_weak = 4;       // compare ETags ignoring the weak (W/) marker
+        string ext_key = 5;        // extended attribute name for IF_EXTENDED_* kinds
+        string ext_value = 6;      // blocking value for IF_EXTENDED_NOT_EQUAL
+        string gate_key = 7;       // IF_EXTENDED_TIME_ELAPSED: only enforce when extended[gate_key] == gate_value
+        string gate_value = 8;     // gate value (e.g. retention mode COMPLIANCE for governance bypass)
+    }
+    repeated Clause clauses = 1;   // all must hold (logical AND)
 }
 
 // Structured error codes for filer entry operations.
@@ -228,6 +292,132 @@ enum FilerError {
     EXISTING_IS_DIRECTORY = 3; // cannot overwrite directory with file
     EXISTING_IS_FILE      = 4; // cannot overwrite file with directory
     ENTRY_ALREADY_EXISTS  = 5; // O_EXCL and entry already exists
+    PRECONDITION_FAILED   = 6; // WriteCondition not satisfied
+}
+
+// ObjectMutation is one entry-level change applied by ObjectTransaction. All
+// mutations of a transaction run under a single per-path lock (the request's
+// lock_key) and in order, so the gateway can describe a multi-entry object
+// operation as one request instead of holding a distributed lock across
+// several RPCs. Data-bearing writes (entries with chunks) should be written
+// before the transaction; mutations here are metadata-scoped.
+message ObjectMutation {
+    enum Type {
+        PUT              = 0; // create or replace the entry (entry field)
+        DELETE           = 1; // delete the entry at directory/name (no error if absent)
+        PATCH_EXTENDED   = 2; // merge set_extended / remove delete_extended on the entry
+        RECOMPUTE_LATEST = 3; // scan a directory and re-point a parent entry (recompute)
+    }
+    Type type = 1;
+    string directory = 2;
+    string name = 3;                       // entry name for DELETE / PATCH_EXTENDED / RECOMPUTE_LATEST (the pointer entry)
+    Entry entry = 4;                        // full entry for PUT
+    map<string, bytes> set_extended = 5;   // PATCH_EXTENDED: keys to set
+    repeated string delete_extended = 6;   // PATCH_EXTENDED: keys to remove
+    bool is_delete_data = 7;               // DELETE: also delete chunk data
+    bool is_recursive = 8;                 // DELETE: recurse into a directory
+    Recompute recompute = 9;               // RECOMPUTE_LATEST parameters
+    bool set_content = 10;                 // PATCH_EXTENDED: replace Entry.content with content
+    bytes content = 11;                    // PATCH_EXTENDED: new Entry.content when set_content
+    bool touch_mtime = 12;                 // PATCH_EXTENDED: set the entry's Mtime to now (e.g. a metadata-replace copy)
+}
+
+// Recompute re-derives a pointer entry (directory/name on the mutation) from the
+// current contents of a scanned directory, atomically under the transaction's
+// lock. It is mechanical: the filer picks the child that sorts first or last by
+// name and copies the requested fields into the pointer; it has no knowledge of
+// what the entries mean. The caller (which does know the versioning scheme)
+// supplies the sort direction and the key mappings. This covers re-pointing the
+// latest version after a specific version is deleted, where the scan must run
+// under the lock.
+message Recompute {
+    string scan_dir = 1;                   // directory whose direct children are scanned
+    bool descending = 2;                   // pick the child that sorts last by name (else first)
+    map<string, string> copy_extended = 3; // pointer extended key -> source extended key on the chosen child
+    string name_to_key = 4;                // if set, store the chosen child's name under this pointer key
+    string size_to_key = 5;                // if set, store the chosen child's FileSize (decimal) under this pointer key
+    string mtime_to_key = 6;               // if set, store the chosen child's Mtime (decimal) under this pointer key
+    string demote_key = 7;                 // if set, stamp demote_value on the prior name_to_key target when it changes
+    bytes demote_value = 8;                // value for demote_key
+    string exclude_name = 9;               // if set, skip this child when scanning (e.g. a version about to be deleted)
+}
+
+// ObjectTransactionRequest applies an ordered list of mutations atomically with
+// respect to other writers of the same object, by holding the filer's per-path
+// lock on lock_key for the whole transaction. The optional condition is checked
+// first, against condition_key when set, else lock_key. Callers set route_key to
+// the object's stable owner ring key; a filer that is not the owner forwards the
+// transaction one hop to the owner, so a stale ring view is tolerated.
+message ObjectTransactionRequest {
+    string lock_key = 1;               // object path to lock and to evaluate the condition against
+    WriteCondition condition = 2;      // optional precondition, checked under the lock
+    repeated ObjectMutation mutations = 3;
+    bool is_from_other_cluster = 4;
+    repeated int32 signatures = 5;
+    string condition_key = 6;          // if set, evaluate the condition against this entry instead of lock_key (still locking lock_key)
+    string route_key = 7;              // ring key identifying the owner filer; a non-owner forwards the whole transaction to it
+    bool is_moved = 8;                 // set on a forwarded transaction so the receiver applies it locally instead of forwarding again
+}
+
+message ObjectTransactionResponse {
+    string error = 1;
+    FilerError error_code = 2;
+}
+
+// PosixLockRange is one advisory byte-range lock. Owner identity is (sid, owner):
+// sid is the mount session, owner the FUSE lock owner within it, so owners from
+// different mounts never alias. end is inclusive (max uint64 = to EOF); is_flock
+// separates the flock and fcntl namespaces, which never conflict.
+message PosixLockRange {
+    uint64 start = 1;
+    uint64 end = 2;
+    uint32 type = 3;       // 1=read, 2=write, 3=unlock
+    uint64 sid = 4;
+    uint64 owner = 5;
+    uint32 pid = 6;        // holder pid, for get_lk reporting only
+    bool is_flock = 7;
+}
+
+// PosixLock routes an advisory lock operation to the inode's owner filer, which
+// holds the authoritative in-memory lock table. key is the inode identity ring
+// key (the file path, or hl:<HardLinkId> for a hardlink) used both to resolve the
+// owner and to index the table. A non-owner filer forwards the request one hop;
+// is_moved bounds it so a stale ring view cannot loop.
+message PosixLockRequest {
+    string key = 1;
+    bool is_moved = 2;
+    PosixLockOp op = 3;
+    PosixLockRange lock = 4;
+    repeated PosixLockRange locks = 5;
+    bool cooling_probe = 6;
+}
+
+enum PosixLockOp {
+    TRY_LOCK = 0;             // grant lock or report conflict (non-blocking)
+    UNLOCK = 1;              // release lock's owner's locks over its range
+    GET_LK = 2;             // report a conflicting lock, if any
+    RELEASE_POSIX_OWNER = 3; // drop the owner's fcntl locks (flush-time)
+    RELEASE_FLOCK_OWNER = 4; // drop the owner's flock locks (release-time)
+    KEEP_ALIVE = 5;         // renew the session's lease on this owner (lock.sid)
+}
+
+message PosixLockResponse {
+    bool granted = 1;            // for TRY_LOCK: whether the lock was granted
+    bool has_conflict = 2;       // whether conflict is populated
+    PosixLockRange conflict = 3; // the blocking lock (TRY_LOCK conflict / GET_LK result)
+}
+
+// ObjectTransactionBatch applies several object transactions in one round trip,
+// each under its own per-path lock and independent of the others (no cross-key
+// atomicity). A caller groups keys that route to the same owner filer and sends
+// one batch per owner, e.g. for a multi-object delete. Each response is parallel
+// to its request.
+message ObjectTransactionBatchRequest {
+    repeated ObjectTransactionRequest transactions = 1;
+}
+
+message ObjectTransactionBatchResponse {
+    repeated ObjectTransactionResponse responses = 1;
 }
 
 message CreateEntryResponse {
@@ -247,6 +437,16 @@ message UpdateEntryResponse {
     SubscribeMetadataResponse metadata_event = 1;
 }
 
+message TouchAccessTimeRequest {
+    string directory = 1;
+    string name = 2;
+    int64 client_atime_ns = 3; // nanoseconds since epoch; filer may override with relatime
+}
+message TouchAccessTimeResponse {
+    int64 persisted_atime_ns = 1; // nanoseconds since epoch; 0 if no update was performed
+    bool updated = 2;
+}
+
 message AppendToEntryRequest {
     string directory = 1;
     string entry_name = 2;
@@ -406,6 +606,7 @@ message SubscribeMetadataRequest {
     repeated string directories = 10;  // exact directory to watch
     bool client_supports_batching = 11; // client can unpack SubscribeMetadataResponse.events
     bool client_supports_metadata_chunks = 12; // client can read log file chunks from volume servers
+    bool client_supports_idle_heartbeat = 13; // server may send empty responses carrying the current time while the client is caught up
 }
 message SubscribeMetadataResponse {
     string directory = 1;

seaweed-volume/src/server/grpc_client.rs

@@ -107,6 +107,57 @@ pub fn build_grpc_endpoint(
     Ok(endpoint)
 }
 
+/// Parse a SeaweedFS server address (`"ip:port.grpcPort"` or
+/// `"ip:port"`) into the `host:grpcPort` form `build_grpc_endpoint`
+/// expects. With the trailing `.grpcPort` segment, that segment IS
+/// the gRPC port; without it, the gRPC port is `port + 10000`
+/// (SeaweedFS's HTTP↔gRPC port-offset convention).
+///
+/// Shared between `grpc_server.rs` and the distributed-EC-read path
+/// in `store_ec.rs` — keep this as the single source of truth so the
+/// HTTP↔gRPC port translation can't drift between callers.
+pub fn parse_grpc_address(source: &str) -> Result<String, String> {
+    let colon_idx = source
+        .rfind(':')
+        .ok_or_else(|| format!("cannot parse address: {}", source))?;
+    let host = &source[..colon_idx];
+    let port_part = &source[colon_idx + 1..];
+
+    if let Some(dot_idx) = port_part.rfind('.') {
+        // Format: "ip:port.grpcPort". Validate BOTH ports as u16
+        // so a malformed HTTP port (e.g. `host:abc.18080`) is
+        // rejected here rather than tripping a downstream
+        // `build_grpc_endpoint` URI parse failure with a less
+        // useful error.
+        let http_port = &port_part[..dot_idx];
+        let grpc_port = &port_part[dot_idx + 1..];
+        http_port
+            .parse::<u16>()
+            .map_err(|e| format!("invalid http port {:?}: {}", http_port, e))?;
+        grpc_port
+            .parse::<u16>()
+            .map_err(|e| format!("invalid grpc port {:?}: {}", grpc_port, e))?;
+        return Ok(format!("{}:{}", host, grpc_port));
+    }
+
+    // Format: "ip:port" → grpc = port + 10000. Reject inputs whose
+    // implicit grpc port would overflow the TCP port range (e.g.
+    // `host:60000` produces 70000 — invalid). Without this check
+    // the cast silently wraps and the endpoint call later fails
+    // with an opaque connection error.
+    let port: u16 = port_part
+        .parse()
+        .map_err(|e| format!("invalid port {:?}: {}", port_part, e))?;
+    let grpc_port = port as u32 + 10000;
+    if grpc_port > u16::MAX as u32 {
+        return Err(format!(
+            "implicit grpc port out of range: {} + 10000 = {}",
+            port, grpc_port
+        ));
+    }
+    Ok(format!("{}:{}", host, grpc_port))
+}
+
 #[cfg(test)]
 mod tests {
     use super::{build_grpc_endpoint, grpc_endpoint_uri, load_outgoing_grpc_tls};
@@ -203,4 +254,50 @@ mod tests {
         let endpoint = build_grpc_endpoint("127.0.0.1:19333", None).unwrap();
         assert_eq!(endpoint.uri().scheme_str(), Some("http"));
     }
+
+    #[test]
+    fn test_parse_grpc_address_dotted_form() {
+        use super::parse_grpc_address;
+        assert_eq!(
+            parse_grpc_address("127.0.0.1:8080.18080").unwrap(),
+            "127.0.0.1:18080"
+        );
+    }
+
+    #[test]
+    fn test_parse_grpc_address_implicit_form_adds_10000() {
+        use super::parse_grpc_address;
+        assert_eq!(
+            parse_grpc_address("127.0.0.1:8080").unwrap(),
+            "127.0.0.1:18080"
+        );
+    }
+
+    #[test]
+    fn test_parse_grpc_address_rejects_non_numeric_http_port_in_dotted_form() {
+        use super::parse_grpc_address;
+        let err = parse_grpc_address("host:abc.18080").unwrap_err();
+        assert!(err.contains("invalid http port"), "{}", err);
+    }
+
+    #[test]
+    fn test_parse_grpc_address_rejects_non_numeric_grpc_port_in_dotted_form() {
+        use super::parse_grpc_address;
+        let err = parse_grpc_address("host:8080.xyz").unwrap_err();
+        assert!(err.contains("invalid grpc port"), "{}", err);
+    }
+
+    #[test]
+    fn test_parse_grpc_address_rejects_implicit_port_that_overflows() {
+        use super::parse_grpc_address;
+        let err = parse_grpc_address("127.0.0.1:60000").unwrap_err();
+        assert!(err.contains("out of range"), "{}", err);
+    }
+
+    #[test]
+    fn test_parse_grpc_address_rejects_input_without_colon() {
+        use super::parse_grpc_address;
+        let err = parse_grpc_address("hostname").unwrap_err();
+        assert!(err.contains("cannot parse"), "{}", err);
+    }
 }

seaweed-volume/src/server/grpc_server.rs

@@ -1511,9 +1511,11 @@ impl VolumeServer for VolumeGrpcService {
                             // EcVolume holds fds on the same inodes, so overwriting
                             // corrupts live readers.
                             if store.has_ec_volume(VolumeId(info.volume_id)) {
+                                let mounted_disks =
+                                    store.find_ec_volume_disk_ids(VolumeId(info.volume_id));
                                 resp_error = Some(format!(
-                                    "ec volume {} is mounted; unmount before ReceiveFile",
-                                    info.volume_id
+                                    "ec volume {} is mounted on disk_ids:{:?}; unmount before ReceiveFile",
+                                    info.volume_id, mounted_disks
                                 ));
                                 break;
                             }
@@ -4051,29 +4053,12 @@ async fn ping_filer_target(
     Ok(resp.into_inner().start_time_ns)
 }
 
-/// Parse a SeaweedFS server address ("ip:port.grpcPort" or "ip:port") into a gRPC address.
-fn parse_grpc_address(source: &str) -> Result<String, String> {
-    if let Some(colon_idx) = source.rfind(':') {
-        let port_part = &source[colon_idx + 1..];
-        if let Some(dot_idx) = port_part.rfind('.') {
-            // Format: "ip:port.grpcPort"
-            let host = &source[..colon_idx];
-            let grpc_port = &port_part[dot_idx + 1..];
-            grpc_port
-                .parse::<u16>()
-                .map_err(|e| format!("invalid grpc port: {}", e))?;
-            return Ok(format!("{}:{}", host, grpc_port));
-        }
-        // Format: "ip:port" → grpc = port + 10000
-        let port: u16 = port_part
-            .parse()
-            .map_err(|e| format!("invalid port: {}", e))?;
-        let grpc_port = port as u32 + 10000;
-        let host = &source[..colon_idx];
-        return Ok(format!("{}:{}", host, grpc_port));
-    }
-    Err(format!("cannot parse address: {}", source))
-}
+// parse_grpc_address moved to super::grpc_client::parse_grpc_address
+// for sharing with the distributed-EC-read path in server/store_ec.rs.
+// In-file callers below still write `parse_grpc_address(...)`; this
+// `use` makes them resolve to the new home without churning every
+// call site.
+use super::grpc_client::parse_grpc_address;
 
 /// Set the modification time of a file from nanoseconds since Unix epoch.
 fn set_file_mtime(path: &str, modified_ts_ns: i64) {

seaweed-volume/src/server/handlers.rs

@@ -838,8 +838,6 @@ pub struct ReadQueryParams {
     pub response_content_disposition: Option<String>,
     /// Pretty print JSON response
     pub pretty: Option<String>,
-    /// JSONP callback function name
-    pub callback: Option<String>,
 }
 
 // ============================================================================
@@ -1019,43 +1017,44 @@ async fn get_or_head_handler_inner(
 
     if has_ec_volume && !has_volume {
         // ---- EC volume read path (always full read, no streaming) ----
-        let store = state.store.read().unwrap();
-        match store.find_ec_volume(vid) {
-            Some(ecv) => match ecv.read_ec_shard_needle(needle_id) {
-                Ok(Some(ec_needle)) => {
-                    n = ec_needle;
-                }
-                Ok(None) => {
-                    metrics::HANDLER_COUNTER
-                        .with_label_values(&[metrics::ERROR_GET_NOT_FOUND])
-                        .inc();
-                    return StatusCode::NOT_FOUND.into_response();
-                }
-                Err(e) => {
-                    if e.kind() == std::io::ErrorKind::NotFound {
-                        metrics::HANDLER_COUNTER
-                            .with_label_values(&[metrics::ERROR_GET_NOT_FOUND])
-                            .inc();
-                        return StatusCode::NOT_FOUND.into_response();
-                    }
-                    metrics::HANDLER_COUNTER
-                        .with_label_values(&[metrics::ERROR_GET_INTERNAL])
-                        .inc();
-                    return (
-                        StatusCode::INTERNAL_SERVER_ERROR,
-                        format!("read ec error: {}", e),
-                    )
-                        .into_response();
-                }
-            },
-            None => {
+        //
+        // The distributed read path already does a local-first pass
+        // in its Snapshot phase under the same store read lock the
+        // legacy code would have taken — so calling it directly
+        // serves both the "all shards local" fast case and the
+        // "some intervals need peer fetch + reconstruct" general
+        // case without paying for the local interval reads twice.
+        match crate::server::store_ec::read_ec_shard_needle_distributed(
+            &state, vid, needle_id,
+        )
+        .await
+        {
+            Ok(Some(ec_needle)) => {
+                n = ec_needle;
+            }
+            Ok(None) => {
                 metrics::HANDLER_COUNTER
                     .with_label_values(&[metrics::ERROR_GET_NOT_FOUND])
                     .inc();
                 return StatusCode::NOT_FOUND.into_response();
             }
+            Err(e) => {
+                let kind = if e.kind() == std::io::ErrorKind::NotFound {
+                    metrics::ERROR_GET_NOT_FOUND
+                } else {
+                    metrics::ERROR_GET_INTERNAL
+                };
+                metrics::HANDLER_COUNTER.with_label_values(&[kind]).inc();
+                if e.kind() == std::io::ErrorKind::NotFound {
+                    return StatusCode::NOT_FOUND.into_response();
+                }
+                return (
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    format!("ec read: {}", e),
+                )
+                    .into_response();
+            }
         }
-        drop(store);
 
         // Validate cookie (matches Go behavior after ReadEcShardNeedle)
         if n.cookie != cookie {
@@ -3401,10 +3400,6 @@ fn json_response_with_params<T: Serialize>(
     let is_pretty = params
         .and_then(|params| params.pretty.as_ref())
         .is_some_and(|value| !value.is_empty());
-    let callback = params
-        .and_then(|params| params.callback.as_ref())
-        .filter(|value| !value.is_empty())
-        .cloned();
 
     let json_body = if is_pretty {
         to_pretty_json(body)
@@ -3412,24 +3407,15 @@ fn json_response_with_params<T: Serialize>(
         serde_json::to_string(body).unwrap()
     };
 
-    if let Some(callback) = callback {
-        Response::builder()
-            .status(status)
-            .header(header::CONTENT_TYPE, "application/javascript")
-            .body(Body::from(format!("{}({})", callback, json_body)))
-            .unwrap()
-    } else {
-        Response::builder()
-            .status(status)
-            .header(header::CONTENT_TYPE, "application/json")
-            .body(Body::from(json_body))
-            .unwrap()
-    }
+    Response::builder()
+        .status(status)
+        .header(header::CONTENT_TYPE, "application/json")
+        .header("X-Content-Type-Options", "nosniff")
+        .body(Body::from(json_body))
+        .unwrap()
 }
 
-/// Return a JSON error response with optional query string for pretty/JSONP support.
-/// Supports `?pretty=<any non-empty value>` for pretty-printed JSON and `?callback=fn` for JSONP,
-/// matching Go's writeJsonError behavior.
+/// Return a JSON error response, honoring `?pretty=<any non-empty value>` for pretty-printed JSON.
 pub(super) fn json_error_with_query(
     status: StatusCode,
     msg: impl Into<String>,
@@ -3437,18 +3423,10 @@ pub(super) fn json_error_with_query(
 ) -> Response {
     let body = serde_json::json!({"error": msg.into()});
 
-    let (is_pretty, callback) = if let Some(q) = query {
-        let pretty = q
-            .split('&')
-            .any(|p| p.starts_with("pretty=") && p.len() > "pretty=".len());
-        let cb = q
-            .split('&')
-            .find_map(|p| p.strip_prefix("callback="))
-            .map(|s| s.to_string());
-        (pretty, cb)
-    } else {
-        (false, None)
-    };
+    let is_pretty = query.is_some_and(|q| {
+        q.split('&')
+            .any(|p| p.starts_with("pretty=") && p.len() > "pretty=".len())
+    });
 
     let json_body = if is_pretty {
         to_pretty_json(&body)
@@ -3456,35 +3434,19 @@ pub(super) fn json_error_with_query(
         serde_json::to_string(&body).unwrap()
     };
 
-    if let Some(cb) = callback {
-        let jsonp = format!("{}({})", cb, json_body);
-        Response::builder()
-            .status(status)
-            .header(header::CONTENT_TYPE, "application/javascript")
-            .body(Body::from(jsonp))
-            .unwrap()
-    } else {
-        Response::builder()
-            .status(status)
-            .header(header::CONTENT_TYPE, "application/json")
-            .body(Body::from(json_body))
-            .unwrap()
-    }
+    Response::builder()
+        .status(status)
+        .header(header::CONTENT_TYPE, "application/json")
+        .header("X-Content-Type-Options", "nosniff")
+        .body(Body::from(json_body))
+        .unwrap()
 }
 
-/// Return a JSON response with optional pretty/JSONP support from raw query string.
-/// Matches Go's writeJsonQuiet behavior for write success responses.
+/// Return a JSON response honoring `?pretty=<any non-empty value>` from a raw query string.
 fn json_result_with_query<T: Serialize>(status: StatusCode, body: &T, query: &str) -> Response {
-    let (is_pretty, callback) = {
-        let pretty = query
-            .split('&')
-            .any(|p| p.starts_with("pretty=") && p.len() > "pretty=".len());
-        let cb = query
-            .split('&')
-            .find_map(|p| p.strip_prefix("callback="))
-            .map(|s| s.to_string());
-        (pretty, cb)
-    };
+    let is_pretty = query
+        .split('&')
+        .any(|p| p.starts_with("pretty=") && p.len() > "pretty=".len());
 
     let json_body = if is_pretty {
         to_pretty_json(body)
@@ -3492,20 +3454,12 @@ fn json_result_with_query<T: Serialize>(status: StatusCode, body: &T, query: &st
         serde_json::to_string(body).unwrap()
     };
 
-    if let Some(cb) = callback {
-        let jsonp = format!("{}({})", cb, json_body);
-        Response::builder()
-            .status(status)
-            .header(header::CONTENT_TYPE, "application/javascript")
-            .body(Body::from(jsonp))
-            .unwrap()
-    } else {
-        Response::builder()
-            .status(status)
-            .header(header::CONTENT_TYPE, "application/json")
-            .body(Body::from(json_body))
-            .unwrap()
-    }
+    Response::builder()
+        .status(status)
+        .header(header::CONTENT_TYPE, "application/json")
+        .header("X-Content-Type-Options", "nosniff")
+        .body(Body::from(json_body))
+        .unwrap()
 }
 
 /// Extract JWT token from query param, Authorization header, or Cookie.

seaweed-volume/src/server/heartbeat.rs

@@ -344,6 +344,12 @@ fn diff_ec_shard_delta_messages(
         if !current.contains_key(key) {
             let mut deleted = message.clone();
             deleted.shard_sizes = vec![0];
+            tracing::info!(
+                volume_id = deleted.id,
+                disk_id = deleted.disk_id,
+                ec_index_bits = deleted.ec_index_bits,
+                "deletes ec shards"
+            );
             deleted_ec_shards.push(deleted);
         }
     }

seaweed-volume/src/server/mod.rs

@@ -9,6 +9,7 @@ pub mod memory_status;
 pub mod profiling;
 pub mod request_id;
 pub mod server_stats;
+pub mod store_ec;
 pub mod ui;
 pub mod volume_server;
 pub mod write_queue;

seaweed-volume/src/server/store_ec.rs

@@ -0,0 +1,678 @@
+//! Distributed EC read path. Mirror of `weed/storage/store_ec.go`'s
+//! `readEcShardIntervals` → `readOneEcShardInterval` →
+//! `readRemoteEcShardInterval` → `recoverOneRemoteEcShardInterval`
+//! chain.
+//!
+//! The existing `EcVolume::read_ec_shard_needle` reads only locally-
+//! mounted shards and returns `NotFound` if any interval requires a
+//! shard held on a peer server. In a standard RS(10,4)-across-14
+//! deployment each server holds one shard, so every read needs >=9
+//! peer fetches. This module fills the gap by:
+//!
+//!   1. Locating the needle in `.ecx` (under the Store read lock) and
+//!      computing the per-interval (shard_id, shard_offset, size).
+//!   2. Reading the local-resident intervals while still holding the
+//!      lock — same path the local-only helper uses.
+//!   3. Dropping the lock and, for any remaining intervals, fetching
+//!      from peer volume servers via `VolumeEcShardRead`. If the
+//!      direct peer read fails, fan-out reads to other shards at the
+//!      same (shard_offset, size) and rebuild the missing shard via
+//!      Reed-Solomon — exactly Go's flow.
+//!   4. Refreshing the per-EcVolume `shard_locations` cache from the
+//!      master's `LookupEcVolume` RPC when the cached map is stale.
+//!
+//! All gRPC IO is async; the file IO portion runs under the sync
+//! Store read lock, matching Go's `readLocalEcShardInterval`. The
+//! cache write-back briefly reacquires the EcVolume's internal
+//! `RwLock` so we do not contend with the Store-level lock at all.
+
+use std::collections::HashMap;
+use std::io;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use futures::future::join_all;
+use reed_solomon_erasure::galois_8::ReedSolomon;
+use tonic::Request;
+
+use crate::pb::master_pb::{self, seaweed_client::SeaweedClient, LookupEcVolumeRequest};
+use crate::pb::volume_server_pb::{
+    volume_server_client::VolumeServerClient, VolumeEcShardReadRequest,
+};
+use crate::server::grpc_client::{build_grpc_endpoint, parse_grpc_address, GRPC_MAX_MESSAGE_SIZE};
+use crate::server::request_id::outgoing_request_id_interceptor;
+use crate::server::volume_server::VolumeServerState;
+use crate::storage::erasure_coding::ec_shard::ShardId;
+use crate::storage::needle::needle::{get_actual_size, Needle};
+use crate::storage::types::*;
+
+/// One interval's data after Phase A.
+enum IntervalResult {
+    /// Already read from a locally-mounted shard.
+    Local(Vec<u8>),
+    /// Shard not local (or local read failed). Must be fetched.
+    NeedRemote {
+        shard_id: ShardId,
+        shard_offset: i64,
+        size: usize,
+    },
+}
+
+/// Snapshot extracted under the Store read lock so Phases B/C can run
+/// without holding any sync lock across `.await`.
+struct Snapshot {
+    data_shards: u32,
+    parity_shards: u32,
+    version: Version,
+    actual_size: usize,
+    offset: Offset,
+    size_for_parse: Size,
+    intervals: Vec<IntervalResult>,
+    cached_locations: HashMap<ShardId, Vec<String>>,
+    cache_refreshed_at: Option<Instant>,
+}
+
+/// Top-level entry point. Returns `Ok(None)` for "not found" (matches
+/// Go's `ReadEcShardNeedle`); errors propagate as `io::Error`.
+pub async fn read_ec_shard_needle_distributed(
+    state: &Arc<VolumeServerState>,
+    vid: VolumeId,
+    needle_id: NeedleId,
+) -> io::Result<Option<Needle>> {
+    // Phase A — under the Store read lock, locate the needle, compute
+    // intervals, and read any locally-mounted shard intervals. We must
+    // not `.await` while holding this guard (std::sync::RwLockReadGuard
+    // is !Send).
+    let snapshot = match snapshot_under_lock(state, vid, needle_id)? {
+        Some(s) => s,
+        None => return Ok(None),
+    };
+
+    // Phase B — refresh the shard_locations cache from the master if
+    // it is stale. Do this lazily: if every needed interval was read
+    // locally we can skip the master RPC entirely.
+    let any_remote = snapshot
+        .intervals
+        .iter()
+        .any(|r| matches!(r, IntervalResult::NeedRemote { .. }));
+    let total_shards = (snapshot.data_shards + snapshot.parity_shards) as usize;
+
+    let mut shard_locations = snapshot.cached_locations.clone();
+    if any_remote
+        && needs_refresh(
+            &shard_locations,
+            snapshot.cache_refreshed_at,
+            snapshot.data_shards as usize,
+            total_shards,
+        )
+    {
+        match cached_lookup_ec_shard_locations(state, vid).await {
+            Ok(fresh) => {
+                shard_locations = fresh.clone();
+                write_back_shard_locations(state, vid, fresh);
+            }
+            Err(e) => {
+                // Lookup failed — proceed with cached values. If cache
+                // is empty, the remote fetch below will fail and we
+                // surface a NotFound (matching Go's behavior when no
+                // locations are known).
+                tracing::warn!(
+                    "ec lookup failed for volume {}: {} — using cached locations ({} entries)",
+                    vid.0,
+                    e,
+                    shard_locations.len(),
+                );
+            }
+        }
+    }
+
+    // Phase C — fetch missing intervals, reconstructing when the
+    // direct peer read fails.
+    let mut assembled: Vec<Vec<u8>> = Vec::with_capacity(snapshot.intervals.len());
+    for res in snapshot.intervals {
+        match res {
+            IntervalResult::Local(buf) => assembled.push(buf),
+            IntervalResult::NeedRemote {
+                shard_id,
+                shard_offset,
+                size,
+            } => {
+                let buf = fetch_one_interval(
+                    state,
+                    vid,
+                    needle_id,
+                    shard_id,
+                    shard_offset,
+                    size,
+                    &shard_locations,
+                    snapshot.data_shards as usize,
+                    snapshot.parity_shards as usize,
+                )
+                .await?;
+                assembled.push(buf);
+            }
+        }
+    }
+
+    // Phase D — assemble and parse the Needle. Mirrors the tail of
+    // `EcVolume::read_ec_shard_needle`.
+    let mut bytes = Vec::with_capacity(snapshot.actual_size);
+    for chunk in assembled {
+        bytes.extend_from_slice(&chunk);
+    }
+    bytes.truncate(snapshot.actual_size);
+    if bytes.len() < snapshot.actual_size {
+        return Err(io::Error::new(
+            io::ErrorKind::UnexpectedEof,
+            format!(
+                "read {} bytes but need {} for needle {}",
+                bytes.len(),
+                snapshot.actual_size,
+                needle_id
+            ),
+        ));
+    }
+
+    let mut n = Needle::default();
+    n.id = needle_id;
+    n.read_bytes(
+        &bytes,
+        snapshot.offset.to_actual_offset(),
+        snapshot.size_for_parse,
+        snapshot.version,
+    )
+    .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("{}", e)))?;
+    Ok(Some(n))
+}
+
+fn snapshot_under_lock(
+    state: &Arc<VolumeServerState>,
+    vid: VolumeId,
+    needle_id: NeedleId,
+) -> io::Result<Option<Snapshot>> {
+    let store = state.store.read().unwrap();
+    let ecv = match store.find_ec_volume(vid) {
+        Some(v) => v,
+        None => return Ok(None),
+    };
+
+    // Reuse EcVolume::locate_needle for offset/size resolution AND
+    // the per-needle shard-interval math — it's the same routine the
+    // local-only read path uses, so we stay byte-identical on the
+    // shard-size + interval boundaries.
+    let (offset, size, intervals) = match ecv.locate_needle(needle_id)? {
+        Some(v) => v,
+        None => return Ok(None),
+    };
+    if intervals.is_empty() {
+        return Err(io::Error::new(
+            io::ErrorKind::InvalidData,
+            "no intervals for needle",
+        ));
+    }
+    let actual = get_actual_size(size, ecv.version);
+
+    // Phase A.local: for each interval read the local shard if we
+    // hold it; otherwise fall through to remote. We accumulate the
+    // results in interval order so the assembly step is just a
+    // concat.
+    let mut interval_results = Vec::with_capacity(intervals.len());
+    for interval in &intervals {
+        let (shard_id, shard_offset) = interval.to_shard_id_and_offset(ecv.data_shards);
+        let buf_size = interval.size as usize;
+        let local = ecv
+            .shards
+            .get(shard_id as usize)
+            .and_then(|s| s.as_ref());
+        match local {
+            Some(shard) => {
+                let mut buf = vec![0u8; buf_size];
+                match shard.read_at(&mut buf, shard_offset as u64) {
+                    Ok(n) if n == buf_size => {
+                        interval_results.push(IntervalResult::Local(buf));
+                    }
+                    _ => interval_results.push(IntervalResult::NeedRemote {
+                        shard_id,
+                        shard_offset,
+                        size: buf_size,
+                    }),
+                }
+            }
+            None => interval_results.push(IntervalResult::NeedRemote {
+                shard_id,
+                shard_offset,
+                size: buf_size,
+            }),
+        }
+    }
+
+    let cached_locations = ecv.shard_locations.read().unwrap().clone();
+    let cache_refreshed_at = *ecv.shard_locations_refresh_time.lock().unwrap();
+
+    Ok(Some(Snapshot {
+        data_shards: ecv.data_shards,
+        parity_shards: ecv.parity_shards,
+        version: ecv.version,
+        actual_size: actual as usize,
+        offset,
+        size_for_parse: size,
+        intervals: interval_results,
+        cached_locations,
+        cache_refreshed_at,
+    }))
+}
+
+/// Master `LookupEcVolume` freshness rules — match Go's
+/// `cachedLookupEcShardLocations` thresholds in store_ec.go.
+fn needs_refresh(
+    locations: &HashMap<ShardId, Vec<String>>,
+    refreshed_at: Option<Instant>,
+    data_shards: usize,
+    total_shards: usize,
+) -> bool {
+    let now = Instant::now();
+    let age = match refreshed_at {
+        Some(t) => now.saturating_duration_since(t),
+        None => return true,
+    };
+    let shard_count = locations.len();
+    if shard_count < data_shards && age < Duration::from_secs(11) {
+        return false;
+    }
+    if shard_count == total_shards && age < Duration::from_secs(37 * 60) {
+        return false;
+    }
+    if shard_count >= data_shards && age < Duration::from_secs(7 * 60) {
+        return false;
+    }
+    true
+}
+
+async fn cached_lookup_ec_shard_locations(
+    state: &Arc<VolumeServerState>,
+    vid: VolumeId,
+) -> io::Result<HashMap<ShardId, Vec<String>>> {
+    let master = {
+        let live = state.current_master_url.read().await.clone();
+        if !live.is_empty() {
+            live
+        } else {
+            state.master_url.clone()
+        }
+    };
+    if master.is_empty() {
+        return Err(io::Error::new(
+            io::ErrorKind::Other,
+            "no master configured for ec shard lookup",
+        ));
+    }
+
+    let grpc_addr = parse_grpc_address(&master)
+        .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
+    let endpoint = build_grpc_endpoint(&grpc_addr, state.outgoing_grpc_tls.as_ref())
+        .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
+    let channel = endpoint
+        .connect_timeout(Duration::from_secs(5))
+        .timeout(Duration::from_secs(10))
+        .connect()
+        .await
+        .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("master connect: {}", e)))?;
+
+    let mut client = SeaweedClient::with_interceptor(channel, outgoing_request_id_interceptor)
+        .max_decoding_message_size(GRPC_MAX_MESSAGE_SIZE)
+        .max_encoding_message_size(GRPC_MAX_MESSAGE_SIZE);
+
+    let resp = client
+        .lookup_ec_volume(Request::new(LookupEcVolumeRequest { volume_id: vid.0 }))
+        .await
+        .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("lookup_ec_volume: {}", e)))?;
+    let resp = resp.into_inner();
+
+    let mut out = HashMap::new();
+    for entry in resp.shard_id_locations {
+        let addrs: Vec<String> = entry
+            .locations
+            .iter()
+            .map(format_location_as_server_address)
+            .collect();
+        out.insert(entry.shard_id as ShardId, addrs);
+    }
+    Ok(out)
+}
+
+fn write_back_shard_locations(
+    state: &Arc<VolumeServerState>,
+    vid: VolumeId,
+    locations: HashMap<ShardId, Vec<String>>,
+) {
+    let store = state.store.read().unwrap();
+    if let Some(ecv) = store.find_ec_volume(vid) {
+        // Atomic swap + freshness stamp so a concurrent reader sees
+        // either the prior cache or the fresh one — never an
+        // intermediate half-replaced map with the freshness flag
+        // already flipped.
+        ecv.replace_shard_locations(locations);
+    }
+}
+
+/// Build a SeaweedFS-style `host:httpPort.grpcPort` address from a
+/// master `Location` so the result is what `parse_grpc_address` (and
+/// the heartbeat path) already understand.
+fn format_location_as_server_address(loc: &master_pb::Location) -> String {
+    let raw = loc
+        .url
+        .trim_start_matches("http://")
+        .trim_start_matches("https://");
+    if loc.grpc_port > 0 {
+        if let Some((host, http_port)) = raw.rsplit_once(':') {
+            return format!("{}:{}.{}", host, http_port, loc.grpc_port);
+        }
+    }
+    raw.to_string()
+}
+
+/// Try direct peer read; on failure, reconstruct via Reed-Solomon
+/// from the other shards. Mirrors `readOneEcShardInterval`'s tail.
+async fn fetch_one_interval(
+    state: &Arc<VolumeServerState>,
+    vid: VolumeId,
+    needle_id: NeedleId,
+    shard_id: ShardId,
+    shard_offset: i64,
+    size: usize,
+    shard_locations: &HashMap<ShardId, Vec<String>>,
+    data_shards: usize,
+    parity_shards: usize,
+) -> io::Result<Vec<u8>> {
+    // Direct peer read against the cached locations for this shard.
+    if let Some(sources) = shard_locations.get(&shard_id) {
+        if !sources.is_empty() {
+            match read_remote_ec_shard_interval(
+                state,
+                sources,
+                vid,
+                needle_id,
+                shard_id,
+                shard_offset,
+                size,
+            )
+            .await
+            {
+                Ok(buf) => return Ok(buf),
+                Err(e) => {
+                    tracing::debug!(
+                        "direct read ec shard {}.{} from {:?} failed: {} — will reconstruct",
+                        vid.0,
+                        shard_id,
+                        sources,
+                        e
+                    );
+                }
+            }
+        }
+    }
+
+    // Reconstruct: fan-out reads to every other shard at the same
+    // (shard_offset, size). Mirrors `recoverOneRemoteEcShardInterval`.
+    recover_one_remote_ec_shard_interval(
+        state,
+        vid,
+        needle_id,
+        shard_id,
+        shard_offset,
+        size,
+        shard_locations,
+        data_shards,
+        parity_shards,
+    )
+    .await
+}
+
+async fn read_remote_ec_shard_interval(
+    state: &Arc<VolumeServerState>,
+    sources: &[String],
+    vid: VolumeId,
+    needle_id: NeedleId,
+    shard_id: ShardId,
+    shard_offset: i64,
+    size: usize,
+) -> io::Result<Vec<u8>> {
+    let mut last_err: Option<io::Error> = None;
+    for src in sources {
+        match do_read_remote_ec_shard_interval(
+            state,
+            src,
+            vid,
+            needle_id,
+            shard_id,
+            shard_offset,
+            size,
+        )
+        .await
+        {
+            Ok(buf) => return Ok(buf),
+            Err(e) => last_err = Some(e),
+        }
+    }
+    Err(last_err.unwrap_or_else(|| {
+        io::Error::new(
+            io::ErrorKind::NotFound,
+            format!("no source for ec shard {}.{}", vid.0, shard_id),
+        )
+    }))
+}
+
+async fn do_read_remote_ec_shard_interval(
+    state: &Arc<VolumeServerState>,
+    source: &str,
+    vid: VolumeId,
+    needle_id: NeedleId,
+    shard_id: ShardId,
+    shard_offset: i64,
+    size: usize,
+) -> io::Result<Vec<u8>> {
+    let grpc_addr =
+        parse_grpc_address(source).map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
+    let endpoint = build_grpc_endpoint(&grpc_addr, state.outgoing_grpc_tls.as_ref())
+        .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
+    let channel = endpoint
+        .connect_timeout(Duration::from_secs(5))
+        .timeout(Duration::from_secs(30))
+        .connect()
+        .await
+        .map_err(|e| {
+            io::Error::new(
+                io::ErrorKind::Other,
+                format!("connect to {}: {}", source, e),
+            )
+        })?;
+
+    // TODO(grpc-jwt): clusters with `jwt.signing.key` configured will
+    // reject peer-to-peer VolumeEcShardRead calls until the Rust
+    // crate grows an outgoing-JWT interceptor. The gap is shared
+    // with every other peer gRPC call from this binary
+    // (`copy_file_from_source`, `batch_delete`, …) — handling it
+    // here in isolation would split the credential plumbing across
+    // call sites. Re-visit when outgoing JWT signing lands as a
+    // server-wide helper.
+    let mut client = VolumeServerClient::with_interceptor(channel, outgoing_request_id_interceptor)
+        .max_decoding_message_size(GRPC_MAX_MESSAGE_SIZE)
+        .max_encoding_message_size(GRPC_MAX_MESSAGE_SIZE);
+
+    let req = VolumeEcShardReadRequest {
+        volume_id: vid.0,
+        shard_id: shard_id as u32,
+        offset: shard_offset,
+        size: size as i64,
+        file_key: needle_id.0,
+    };
+    let resp = client
+        .volume_ec_shard_read(Request::new(req))
+        .await
+        .map_err(|e| {
+            io::Error::new(
+                io::ErrorKind::Other,
+                format!("volume_ec_shard_read {}.{} from {}: {}", vid.0, shard_id, source, e),
+            )
+        })?;
+    let mut stream = resp.into_inner();
+
+    let mut out = Vec::with_capacity(size);
+    while let Some(msg) = stream
+        .message()
+        .await
+        .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("recv: {}", e)))?
+    {
+        if !msg.data.is_empty() {
+            out.extend_from_slice(&msg.data);
+        }
+    }
+    if out.len() < size {
+        return Err(io::Error::new(
+            io::ErrorKind::UnexpectedEof,
+            format!(
+                "short read from {} for ec shard {}.{}: got {} want {}",
+                source,
+                vid.0,
+                shard_id,
+                out.len(),
+                size
+            ),
+        ));
+    }
+    out.truncate(size);
+    Ok(out)
+}
+
+async fn recover_one_remote_ec_shard_interval(
+    state: &Arc<VolumeServerState>,
+    vid: VolumeId,
+    needle_id: NeedleId,
+    shard_id_to_recover: ShardId,
+    shard_offset: i64,
+    size: usize,
+    shard_locations: &HashMap<ShardId, Vec<String>>,
+    data_shards: usize,
+    parity_shards: usize,
+) -> io::Result<Vec<u8>> {
+    let total_shards = data_shards + parity_shards;
+    let rs = ReedSolomon::new(data_shards, parity_shards).map_err(|e| {
+        io::Error::new(
+            io::ErrorKind::Other,
+            format!("reed-solomon init: {:?}", e),
+        )
+    })?;
+
+    let mut bufs: Vec<Option<Vec<u8>>> = vec![None; total_shards];
+
+    // Phase 0: seed bufs from LOCALLY mounted shards. If this node
+    // already holds enough sibling shards, reconstruction completes
+    // without any peer fan-out — and even with a cold/incomplete
+    // shard_locations cache or a failed master lookup, local
+    // survivors still contribute. Mirrors Go's
+    // recoverOneRemoteEcShardInterval behaviour, which is implicitly
+    // local-aware because the Store fan-out targets ALL known
+    // locations (including the caller's own server address); the
+    // Rust port had been remote-only, so reconstructing with a cold
+    // cache failed even when enough siblings were on disk.
+    {
+        let store = state.store.read().unwrap();
+        if let Some(ecv) = store.find_ec_volume(vid) {
+            for sid in 0..total_shards {
+                if sid as ShardId == shard_id_to_recover {
+                    continue;
+                }
+                if let Some(Some(shard)) = ecv.shards.get(sid) {
+                    let mut buf = vec![0u8; size];
+                    if shard.read_at(&mut buf, shard_offset as u64).map(|n| n == size).unwrap_or(false) {
+                        bufs[sid] = Some(buf);
+                    }
+                }
+            }
+        }
+    }
+
+    // Phase 1: remote fan-out — one task per known shard location
+    // we DON'T already have locally and DON'T need to recover.
+    let mut tasks = Vec::new();
+    for (sid, locs) in shard_locations {
+        if *sid == shard_id_to_recover || locs.is_empty() {
+            continue;
+        }
+        if bufs[*sid as usize].is_some() {
+            continue;
+        }
+        let sid = *sid;
+        let locs = locs.clone();
+        let state = state.clone();
+        tasks.push(async move {
+            let res = read_remote_ec_shard_interval(
+                &state,
+                &locs,
+                vid,
+                needle_id,
+                sid,
+                shard_offset,
+                size,
+            )
+            .await;
+            (sid, res)
+        });
+    }
+    let results = join_all(tasks).await;
+
+    for (sid, res) in results {
+        match res {
+            Ok(buf) => {
+                if (sid as usize) < total_shards {
+                    bufs[sid as usize] = Some(buf);
+                }
+            }
+            Err(e) => {
+                tracing::debug!(
+                    "recover: read {}.{} for needle {} failed: {}",
+                    vid.0,
+                    sid,
+                    needle_id,
+                    e
+                );
+            }
+        }
+    }
+
+    let available = bufs.iter().filter(|b| b.is_some()).count();
+    if available < data_shards {
+        return Err(io::Error::new(
+            io::ErrorKind::Other,
+            format!(
+                "cannot recover ec shard {}.{}: only {} shards available, need at least {}",
+                vid.0, shard_id_to_recover, available, data_shards
+            ),
+        ));
+    }
+
+    rs.reconstruct(&mut bufs).map_err(|e| {
+        io::Error::new(
+            io::ErrorKind::Other,
+            format!(
+                "reed-solomon reconstruct ec shard {}.{}: {:?}",
+                vid.0, shard_id_to_recover, e
+            ),
+        )
+    })?;
+
+    match bufs.into_iter().nth(shard_id_to_recover as usize).flatten() {
+        Some(buf) => Ok(buf),
+        None => Err(io::Error::new(
+            io::ErrorKind::Other,
+            format!(
+                "reconstructed buffer for shard {}.{} missing after RS reconstruct",
+                vid.0, shard_id_to_recover
+            ),
+        )),
+    }
+}
+
+// parse_grpc_address lives in `grpc_client.rs` and is re-exported
+// here via the use above so this module shares a single
+// HTTP↔gRPC port-translation routine with grpc_server.rs.

seaweed-volume/src/storage/erasure_coding/ec_volume.rs

@@ -44,8 +44,17 @@ pub struct EcVolume {
     /// Directory where .ecx/.ecj were actually found (may differ from dir_idx after fallback).
     ecx_actual_dir: String,
     /// Maps shard ID -> list of server addresses where that shard exists.
-    /// Used for distributed EC reads across the cluster.
-    pub shard_locations: HashMap<ShardId, Vec<String>>,
+    /// Used for distributed EC reads across the cluster. Wrapped in
+    /// `RwLock` so the read path can refresh the map (under master
+    /// lookup) without holding the Store write lock — mirrors Go's
+    /// `ShardLocationsLock sync.RWMutex` in `weed/storage/erasure_coding/ec_volume.go`.
+    pub shard_locations: std::sync::RwLock<HashMap<ShardId, Vec<String>>>,
+    /// Wall-clock timestamp of the most recent successful
+    /// `LookupEcVolume` refresh of `shard_locations`. `None` until the
+    /// first refresh. Drives the staleness heuristic in
+    /// `cached_lookup_ec_shard_locations` (mirrors Go's
+    /// `ShardLocationsRefreshTime`).
+    pub shard_locations_refresh_time: std::sync::Mutex<Option<std::time::Instant>>,
     /// EC volume expiration time (unix epoch seconds), set during EC encode from TTL.
     pub expire_at_sec: u64,
 }
@@ -165,7 +174,8 @@ impl EcVolume {
             deleted_needles: RwLock::new(HashSet::new()),
             disk_type: DiskType::default(),
             ecx_actual_dir: dir_idx.to_string(),
-            shard_locations: HashMap::new(),
+            shard_locations: std::sync::RwLock::new(HashMap::new()),
+            shard_locations_refresh_time: std::sync::Mutex::new(None),
             expire_at_sec,
         };
 
@@ -430,17 +440,45 @@ impl EcVolume {
 
     // ---- Shard locations (distributed tracking) ----
 
-    /// Set the list of server addresses for a given shard ID.
-    pub fn set_shard_locations(&mut self, shard_id: ShardId, locations: Vec<String>) {
-        self.shard_locations.insert(shard_id, locations);
+    /// Set the list of server addresses for a single shard ID. Does
+    /// NOT touch `shard_locations_refresh_time` — a per-shard write
+    /// from inside a multi-shard population (e.g. iterating the
+    /// `LookupEcVolume` response shard-by-shard) would otherwise
+    /// flip the staleness flag while the map is still incomplete,
+    /// letting a concurrent reader observe `needs_refresh == false`
+    /// against a half-populated cache and return NotFound for the
+    /// not-yet-inserted shards.
+    ///
+    /// Callers populating the whole cache atomically should use
+    /// [`Self::replace_shard_locations`] instead — it swaps the
+    /// entire map under the write lock and advances the refresh
+    /// timestamp in one step.
+    pub fn set_shard_locations(&self, shard_id: ShardId, locations: Vec<String>) {
+        self.shard_locations
+            .write()
+            .unwrap()
+            .insert(shard_id, locations);
     }
 
-    /// Get the list of server addresses for a given shard ID.
-    pub fn get_shard_locations(&self, shard_id: ShardId) -> &[String] {
+    /// Atomically replace the entire shard-locations map and stamp
+    /// the refresh time. Used by the distributed-read path's
+    /// post-`LookupEcVolume` write-back so the cache transitions
+    /// from old → fresh in a single observable step — concurrent
+    /// readers either see the full prior map or the full new map,
+    /// never an intermediate state with the freshness flag flipped.
+    pub fn replace_shard_locations(&self, locations: HashMap<ShardId, Vec<String>>) {
+        *self.shard_locations.write().unwrap() = locations;
+        *self.shard_locations_refresh_time.lock().unwrap() = Some(std::time::Instant::now());
+    }
+
+    /// Get a cloned list of server addresses for a given shard ID.
+    pub fn get_shard_locations(&self, shard_id: ShardId) -> Vec<String> {
         self.shard_locations
+            .read()
+            .unwrap()
             .get(&shard_id)
-            .map(|v| v.as_slice())
-            .unwrap_or(&[])
+            .cloned()
+            .unwrap_or_default()
     }
 
     // ---- Index operations ----

seaweed-volume/src/storage/mod.rs

@@ -4,6 +4,7 @@ pub mod idx;
 pub mod needle;
 pub mod needle_map;
 pub mod store;
+pub mod store_ec_mirror;
 pub mod store_ec_reconcile;
 pub mod super_block;
 pub mod types;

seaweed-volume/src/storage/needle_map.rs

@@ -212,6 +212,13 @@ impl CompactNeedleMap {
         self.idx_file_offset = offset;
     }
 
+    /// True when an .idx file writer is attached. A read-only load leaves
+    /// this `false` — set_writable() must reattach a writer or subsequent
+    /// puts silently skip the disk append.
+    pub fn has_idx_writer(&self) -> bool {
+        self.idx_file.is_some()
+    }
+
     // ---- Map operations ----
 
     /// Insert or update an entry. Appends to .idx file if present.
@@ -705,6 +712,11 @@ impl RedbNeedleMap {
         self.idx_file_offset = offset;
     }
 
+    /// True when an .idx file writer is attached. See CompactNeedleMap.
+    pub fn has_idx_writer(&self) -> bool {
+        self.idx_file.is_some()
+    }
+
     // ---- Map operations ----
 
     /// Insert or update an entry. Writes to idx file first, then redb.
@@ -1000,6 +1012,14 @@ impl NeedleMap {
         }
     }
 
+    /// True when an .idx file writer is attached.
+    pub fn has_idx_writer(&self) -> bool {
+        match self {
+            NeedleMap::InMemory(nm) => nm.has_idx_writer(),
+            NeedleMap::Redb(nm) => nm.has_idx_writer(),
+        }
+    }
+
     /// Content byte count.
     pub fn content_size(&self) -> u64 {
         match self {

seaweed-volume/src/storage/store.rs

@@ -96,14 +96,16 @@ impl Store {
         // re-loading shards we just cleaned up.
         self.prune_incomplete_ec_with_sibling_dat();
 
-        // After every disk has finished its per-disk EC scan, sweep
-        // the store for shards that live on a disk without local index
-        // files and load them by reaching across to a sibling disk's
-        // .ecx / .ecj / .vif (seaweedfs/seaweedfs#9212 / #9244).
-        // ec.balance / ec.rebuild can move shards onto a destination
-        // node's second disk while leaving the index on the disk that
-        // already held the volume; without this pass those orphan
-        // shards stay invisible to the master.
+        // Physically mirror EC sidecars onto every shard-bearing disk
+        // so each disk mounts self-contained. Must run before the
+        // cross-disk reconciler so the orphan pass can prefer the
+        // local idx_directory.
+        self.mirror_ec_metadata_to_shard_disks();
+
+        // Cross-disk fallback for orphan shards — ec.balance can land
+        // shards on one disk while leaving the index on another. Still
+        // needed after the mirror pass for volumes whose mirror failed
+        // (read-only target, partial copy).
         self.reconcile_ec_shards_across_disks();
 
         Ok(())
@@ -118,6 +120,7 @@ impl Store {
             }
         }
         self.prune_incomplete_ec_with_sibling_dat();
+        self.mirror_ec_metadata_to_shard_disks();
         self.reconcile_ec_shards_across_disks();
     }
 
@@ -699,10 +702,20 @@ impl Store {
         // Walk all locations rather than stopping at the first with the
         // vid — split-disk reconciled volumes can have the same vid on
         // multiple disks, with the target shard on any of them.
-        for loc in &mut self.locations {
-            if loc.has_ec_volume(vid) {
-                loc.unmount_ec_shards(vid, &[shard_id]);
+        for disk_id in 0..self.locations.len() {
+            let has_shard = self.locations[disk_id]
+                .find_ec_volume(vid)
+                .is_some_and(|ec_vol| ec_vol.has_shard(shard_id as u8));
+            if !has_shard {
+                continue;
             }
+            tracing::info!(
+                volume_id = vid.0,
+                shard_id,
+                disk_id,
+                "UnmountEcShards"
+            );
+            self.locations[disk_id].unmount_ec_shards(vid, &[shard_id]);
         }
         // Go returns nil if shard not found (no error)
         Ok(())
@@ -733,6 +746,21 @@ impl Store {
         self.locations.iter().any(|loc| loc.has_ec_volume(vid))
     }
 
+    /// Returns every disk_id on this store that has an EcVolume entry
+    /// for `vid`. Useful for diagnostic logging when a single
+    /// `has_ec_volume` hit hides which disk is actually holding the
+    /// mount (e.g., the ReceiveFile mounted-volume guard).
+    /// Mirrors Go's `Store.FindEcVolumeDiskIds`.
+    pub fn find_ec_volume_disk_ids(&self, vid: VolumeId) -> Vec<u32> {
+        let mut ids = Vec::new();
+        for (idx, loc) in self.locations.iter().enumerate() {
+            if loc.has_ec_volume(vid) {
+                ids.push(idx as u32);
+            }
+        }
+        ids
+    }
+
     /// Returns the index of the disk location that has `(vid, shard_id)`
     /// mounted, if any. Mirrors Go's `Store.findEcShard` and is the
     /// right primitive for read/unmount/delete operations on a single

seaweed-volume/src/storage/store_ec_mirror.rs

@@ -0,0 +1,421 @@
+//! Physical EC sidecar mirroring across disks of the same volume
+//! server. Mirrors `weed/storage/store_ec_mirror.go`.
+
+use std::collections::HashMap;
+use std::fs;
+use std::io::{self, Read, Write};
+use std::path::Path;
+
+use tracing::{info, warn};
+
+use crate::storage::disk_location::{parse_collection_volume_id_pub, DiskLocation};
+use crate::storage::store::Store;
+use crate::storage::types::VolumeId;
+
+// Listed in `EcVolume::new`'s open order.
+const EC_MIRRORED_SIDECARS: &[&str] = &[".ecx", ".ecj", ".vif"];
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+struct EcKey {
+    collection: String,
+    vid: VolumeId,
+}
+
+#[derive(Clone, Debug)]
+struct EcxOwner {
+    location: usize,
+    idx_dir: String,
+    data_dir: String,
+}
+
+impl Store {
+    /// Mirror EC sidecars onto every shard-bearing disk that lacks
+    /// them, so each disk mounts self-contained. Runs before
+    /// `reconcile_ec_shards_across_disks` so the orphan pass can
+    /// prefer the local idx_directory.
+    pub fn mirror_ec_metadata_to_shard_disks(&mut self) {
+        if self.locations.len() < 2 {
+            return;
+        }
+        let owners = self.index_ecx_owners_for_mirror();
+        if owners.is_empty() {
+            return;
+        }
+
+        // Two-pass: gather work under an immutable borrow, then apply
+        // copies under independent mutable borrows.
+        struct Mirror<'a> {
+            target_idx: usize,
+            owner: &'a EcxOwner,
+            collection: String,
+            vid: VolumeId,
+        }
+        let mut mirrors: Vec<Mirror> = Vec::new();
+        for (loc_idx, loc) in self.locations.iter().enumerate() {
+            let orphans = collect_shard_disk_volumes(loc);
+            for (key, _shards) in orphans {
+                let Some(owner) = owners.get(&key) else {
+                    continue;
+                };
+                if owner.location == loc_idx {
+                    continue;
+                }
+                if disk_has_all_sidecars(loc, &key.collection, key.vid) {
+                    continue;
+                }
+                mirrors.push(Mirror {
+                    target_idx: loc_idx,
+                    owner,
+                    collection: key.collection,
+                    vid: key.vid,
+                });
+            }
+        }
+
+        for m in mirrors {
+            let loc_dir = self.locations[m.target_idx].directory.clone();
+            let loc_idx_dir = self.locations[m.target_idx].idx_directory.clone();
+            match mirror_sidecars_for_volume(
+                &m.owner.idx_dir,
+                &m.owner.data_dir,
+                &loc_dir,
+                &loc_idx_dir,
+                &m.collection,
+                m.vid,
+            ) {
+                Ok(0) => {}
+                Ok(copied) => {
+                    info!(
+                        volume_id = m.vid.0,
+                        collection = %m.collection,
+                        from = %m.owner.data_dir,
+                        to = %loc_dir,
+                        copied,
+                        "mirrored EC sidecar(s) for same-disk invariant",
+                    );
+                }
+                Err(e) => {
+                    warn!(
+                        volume_id = m.vid.0,
+                        collection = %m.collection,
+                        from = %m.owner.data_dir,
+                        to = %loc_dir,
+                        error = %e,
+                        "mirror EC sidecars failed; cross-disk fallback will handle this volume",
+                    );
+                }
+            }
+        }
+    }
+
+    // Records both idx_dir (where .ecx was found) and data_dir, so
+    // the mirror can resolve .vif from data_dir even when .ecx lives
+    // in idx_directory.
+    fn index_ecx_owners_for_mirror(&self) -> HashMap<EcKey, EcxOwner> {
+        let mut owners: HashMap<EcKey, EcxOwner> = HashMap::new();
+        for (loc_idx, loc) in self.locations.iter().enumerate() {
+            let mut seen: Vec<&str> = Vec::with_capacity(2);
+            for scan in [loc.idx_directory.as_str(), loc.directory.as_str()] {
+                if scan.is_empty() || seen.contains(&scan) {
+                    continue;
+                }
+                seen.push(scan);
+                let Ok(read) = fs::read_dir(scan) else {
+                    continue;
+                };
+                for ent in read.flatten() {
+                    if ent.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
+                        continue;
+                    }
+                    let name = ent.file_name().to_string_lossy().into_owned();
+                    let Some(base) = name.strip_suffix(".ecx") else {
+                        continue;
+                    };
+                    let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
+                        continue;
+                    };
+                    owners
+                        .entry(EcKey { collection, vid })
+                        .or_insert_with(|| EcxOwner {
+                            location: loc_idx,
+                            idx_dir: scan.to_string(),
+                            data_dir: loc.directory.clone(),
+                        });
+                }
+            }
+        }
+        owners
+    }
+}
+
+// Checks the modern routing and the opposite directory — without
+// that fallback, a destination with a legacy pre-`-dir.idx` .ecx in
+// its data dir would be re-mirrored into idx_directory.
+fn disk_has_all_sidecars(loc: &DiskLocation, collection: &str, vid: VolumeId) -> bool {
+    for ext in EC_MIRRORED_SIDECARS {
+        let primary = sidecar_dest_path(&loc.directory, &loc.idx_directory, collection, vid, ext);
+        if path_is_regular_file(&primary) {
+            continue;
+        }
+        if loc.idx_directory != loc.directory {
+            let (fallback_data, fallback_idx) = if *ext == ".vif" {
+                (loc.idx_directory.as_str(), loc.directory.as_str())
+            } else {
+                (loc.directory.as_str(), loc.directory.as_str())
+            };
+            let fallback = sidecar_dest_path(fallback_data, fallback_idx, collection, vid, ext);
+            if path_is_regular_file(&fallback) {
+                continue;
+            }
+        }
+        return false;
+    }
+    true
+}
+
+fn path_is_regular_file(path: &str) -> bool {
+    fs::metadata(path).map(|m| !m.is_dir()).unwrap_or(false)
+}
+
+// `.ecx`/`.ecj` route to idx_directory, `.vif` to directory.
+fn sidecar_dest_path(
+    data_dir: &str,
+    idx_dir: &str,
+    collection: &str,
+    vid: VolumeId,
+    ext: &str,
+) -> String {
+    let dir = if ext == ".vif" { data_dir } else { idx_dir };
+    if collection.is_empty() {
+        format!("{}/{}{}", dir, vid.0, ext)
+    } else {
+        format!("{}/{}_{}{}", dir, collection, vid.0, ext)
+    }
+}
+
+fn mirror_sidecars_for_volume(
+    src_idx_dir: &str,
+    src_data_dir: &str,
+    dst_data_dir: &str,
+    dst_idx_dir: &str,
+    collection: &str,
+    vid: VolumeId,
+) -> io::Result<usize> {
+    let mut copied = 0usize;
+    for ext in EC_MIRRORED_SIDECARS {
+        let dst = sidecar_dest_path(dst_data_dir, dst_idx_dir, collection, vid, ext);
+        // An existing local copy is authoritative — it may be newer
+        // than the owner's after a delete journal append.
+        if fs::metadata(&dst).is_ok() {
+            continue;
+        }
+
+        let candidates = [
+            sidecar_dest_path(src_data_dir, src_idx_dir, collection, vid, ext),
+            sidecar_dest_path(src_idx_dir, src_data_dir, collection, vid, ext),
+        ];
+        let mut src_path: Option<String> = None;
+        for c in candidates.iter() {
+            if fs::metadata(c).map(|m| !m.is_dir()).unwrap_or(false) {
+                src_path = Some(c.clone());
+                break;
+            }
+        }
+        let Some(src) = src_path else {
+            continue;
+        };
+        copy_sidecar_atomic(Path::new(&src), Path::new(&dst))?;
+        copied += 1;
+    }
+    Ok(copied)
+}
+
+fn copy_sidecar_atomic(src: &Path, dst: &Path) -> io::Result<()> {
+    if let Some(parent) = dst.parent() {
+        fs::create_dir_all(parent)?;
+    }
+    let mut src_file = fs::File::open(src)?;
+    let tmp = {
+        let mut s = dst.as_os_str().to_owned();
+        s.push(".mirror.tmp");
+        std::path::PathBuf::from(s)
+    };
+    let _ = fs::remove_file(&tmp);
+    let mut dst_file = fs::OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(&tmp)?;
+    let mut buf = [0u8; 64 * 1024];
+    loop {
+        let n = src_file.read(&mut buf)?;
+        if n == 0 {
+            break;
+        }
+        dst_file.write_all(&buf[..n])?;
+    }
+    dst_file.sync_all()?;
+    drop(dst_file);
+    if let Err(e) = fs::rename(&tmp, dst) {
+        let _ = fs::remove_file(&tmp);
+        return Err(e);
+    }
+    Ok(())
+}
+
+fn collect_shard_disk_volumes(loc: &DiskLocation) -> HashMap<EcKey, Vec<String>> {
+    let mut out: HashMap<EcKey, Vec<String>> = HashMap::new();
+    let Ok(read) = fs::read_dir(&loc.directory) else {
+        return out;
+    };
+    for ent in read.flatten() {
+        if ent.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
+            continue;
+        }
+        let name = ent.file_name().to_string_lossy().into_owned();
+        let Some(dot) = name.rfind('.') else {
+            continue;
+        };
+        let (base, ext) = name.split_at(dot);
+        if crate::storage::disk_location::is_ec_shard_extension(ext).is_none() {
+            continue;
+        }
+        match ent.metadata() {
+            Ok(meta) if meta.len() > 0 => {}
+            _ => continue,
+        }
+        let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
+            continue;
+        };
+        out.entry(EcKey { collection, vid })
+            .or_default()
+            .push(name);
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::config::MinFreeSpace;
+    use crate::storage::needle_map::NeedleMapKind;
+    use crate::storage::types::DiskType;
+    use crate::storage::volume::{VifEcShardConfig, VifVolumeInfo};
+    use tempfile::TempDir;
+
+    fn plant_shard(dir: &Path, collection: &str, vid: u32, shard_id: u8) {
+        let path = if collection.is_empty() {
+            dir.join(format!("{}.ec{:02}", vid, shard_id))
+        } else {
+            dir.join(format!("{}_{}.ec{:02}", collection, vid, shard_id))
+        };
+        fs::write(&path, b"shard data nonempty").unwrap();
+    }
+
+    fn plant_ecx(dir: &Path, collection: &str, vid: u32, bytes: &[u8]) {
+        let path = dir.join(format!("{}_{}.ecx", collection, vid));
+        fs::write(&path, bytes).unwrap();
+    }
+
+    fn plant_ecj(dir: &Path, collection: &str, vid: u32, bytes: &[u8]) {
+        let path = dir.join(format!("{}_{}.ecj", collection, vid));
+        fs::write(&path, bytes).unwrap();
+    }
+
+    fn plant_vif(dir: &Path, collection: &str, vid: u32, data_shards: u32, parity_shards: u32) {
+        let vif = VifVolumeInfo {
+            version: 3,
+            ec_shard_config: Some(VifEcShardConfig {
+                data_shards,
+                parity_shards,
+            }),
+            ..Default::default()
+        };
+        let path = dir.join(format!("{}_{}.vif", collection, vid));
+        fs::write(&path, serde_json::to_string(&vif).unwrap()).unwrap();
+    }
+
+    fn add_loc(store: &mut Store, dir: &Path) {
+        store
+            .add_location(
+                dir.to_str().unwrap(),
+                dir.to_str().unwrap(),
+                100,
+                DiskType::HardDrive,
+                MinFreeSpace::Percent(0.0),
+                Vec::new(),
+            )
+            .unwrap();
+    }
+
+    #[test]
+    fn mirror_copies_sidecars_to_shard_only_disk() {
+        let tmp = TempDir::new().unwrap();
+        let dir0 = tmp.path().join("data0");
+        let dir1 = tmp.path().join("data1");
+        fs::create_dir_all(&dir0).unwrap();
+        fs::create_dir_all(&dir1).unwrap();
+
+        let collection = "video-recordings";
+        let vid = 4121u32;
+
+        plant_shard(&dir0, collection, vid, 0);
+        plant_shard(&dir0, collection, vid, 12);
+        plant_shard(&dir1, collection, vid, 1);
+
+        let ecx = vec![0xA1u8; 20];
+        let ecj = vec![0xB2u8; 16];
+        plant_ecx(&dir1, collection, vid, &ecx);
+        plant_ecj(&dir1, collection, vid, &ecj);
+        plant_vif(&dir1, collection, vid, 10, 4);
+
+        let mut store = Store::new(NeedleMapKind::InMemory);
+        add_loc(&mut store, &dir0);
+        add_loc(&mut store, &dir1);
+
+        for ext in [".ecx", ".ecj", ".vif"] {
+            let dst = dir0.join(format!("{}_{}{}", collection, vid, ext));
+            assert!(
+                dst.exists(),
+                "mirror did not install sidecar {} on dir0",
+                ext
+            );
+        }
+        let ecx_dst = fs::read(dir0.join(format!("{}_{}.ecx", collection, vid))).unwrap();
+        assert_eq!(ecx_dst, ecx, ".ecx mirrored bytes differ from source");
+        let ecj_dst = fs::read(dir0.join(format!("{}_{}.ecj", collection, vid))).unwrap();
+        assert_eq!(ecj_dst, ecj, ".ecj mirrored bytes differ from source");
+    }
+
+    #[test]
+    fn mirror_preserves_existing_destination_sidecars() {
+        let tmp = TempDir::new().unwrap();
+        let dir0 = tmp.path().join("data0");
+        let dir1 = tmp.path().join("data1");
+        fs::create_dir_all(&dir0).unwrap();
+        fs::create_dir_all(&dir1).unwrap();
+
+        let collection = "video-recordings";
+        let vid = 7777u32;
+
+        plant_shard(&dir0, collection, vid, 0);
+        plant_shard(&dir0, collection, vid, 12);
+        plant_shard(&dir1, collection, vid, 1);
+
+        let ecx_owner = vec![0xC3u8; 20];
+        let ecx_local = vec![0x5Au8; 20];
+        let ecj_bytes = vec![0xD4u8; 16];
+        plant_ecx(&dir1, collection, vid, &ecx_owner);
+        plant_ecj(&dir1, collection, vid, &ecj_bytes);
+        plant_vif(&dir1, collection, vid, 10, 4);
+        plant_ecx(&dir0, collection, vid, &ecx_local);
+        plant_ecj(&dir0, collection, vid, &ecj_bytes);
+        plant_vif(&dir0, collection, vid, 10, 4);
+
+        let mut store = Store::new(NeedleMapKind::InMemory);
+        add_loc(&mut store, &dir0);
+        add_loc(&mut store, &dir1);
+
+        let post = fs::read(dir0.join(format!("{}_{}.ecx", collection, vid))).unwrap();
+        assert_eq!(post, ecx_local, "mirror overwrote dir0's existing .ecx");
+    }
+}

seaweed-volume/src/storage/store_ec_reconcile.rs

@@ -26,6 +26,14 @@ use crate::storage::store::Store;
 use crate::storage::super_block::SUPER_BLOCK_SIZE;
 use crate::storage::types::VolumeId;
 
+pub(crate) fn ec_local_ecx_path(dir: &str, collection: &str, vid: VolumeId) -> String {
+    if collection.is_empty() {
+        format!("{}/{}.ecx", dir, vid.0)
+    } else {
+        format!("{}/{}_{}.ecx", dir, collection, vid.0)
+    }
+}
+
 /// Sibling-disk `.dat` candidate for `prune_incomplete_ec_with_sibling_dat`.
 /// We record both the disk index and the file's size: the size is
 /// consulted before deleting any EC artefacts. A zero-byte or truncated
@@ -76,10 +84,10 @@ impl Store {
             return;
         }
 
-        // Snapshot of orphan shards, keyed by (loc_idx, ec_key) so we
-        // can release the immutable borrow on self.locations before
-        // calling mount_ec_shards_with_idx_dir (which needs &mut).
-        let mut to_load: Vec<(usize, EcKey, Vec<(String, u32)>, EcxOwnerInfo)> = Vec::new();
+        // `use_local_idx` is the post-mirror fast path: when the
+        // mirror already installed sidecars locally, mount against
+        // loc.idx_directory instead of the owner disk.
+        let mut to_load: Vec<(usize, EcKey, Vec<(String, u32)>, EcxOwnerInfo, bool)> = Vec::new();
         for (loc_idx, loc) in self.locations.iter().enumerate() {
             let orphans = collect_orphan_ec_shards(loc, loc_idx);
             for (key, shards) in orphans {
@@ -93,34 +101,57 @@ impl Store {
                     );
                     continue;
                 };
-                if owner.location == loc_idx && owner.idx_dir == loc.idx_directory {
-                    // Normal same-disk case: load_all_ec_shards already
-                    // attempted the mount via `loc.idx_directory` and
-                    // logged the underlying failure. No point retrying
-                    // the same call.
+                let local_ecx = ec_local_ecx_path(&loc.idx_directory, &key.collection, key.vid);
+                let local_ecx_in_data = ec_local_ecx_path(&loc.directory, &key.collection, key.vid);
+                let use_local_idx = std::path::Path::new(&local_ecx).exists()
+                    || std::path::Path::new(&local_ecx_in_data).exists();
+
+                if !use_local_idx
+                    && owner.location == loc_idx
+                    && owner.idx_dir == loc.idx_directory
+                {
+                    // Same-disk no-op: load_all_ec_shards already
+                    // tried and logged the failure.
                     continue;
                 }
-                // Either a cross-disk owner OR a same-disk owner whose
-                // `.ecx` actually lives in `loc.directory` (the legacy
-                // pre-`-dir.idx` layout). The latter wasn't tried by
-                // load_all_ec_shards, which only looked in
-                // `self.idx_directory`, so we still need to retry it
-                // here with the owner's discovered idx_dir.
-                to_load.push((loc_idx, key, shards, owner.clone()));
+                to_load.push((loc_idx, key, shards, owner.clone(), use_local_idx));
             }
         }
 
-        for (loc_idx, key, shards, owner) in to_load {
+        for (loc_idx, key, shards, owner, use_local_idx) in to_load {
             let shard_names: Vec<&str> = shards.iter().map(|(n, _)| n.as_str()).collect();
+            let loc_dir = self.locations[loc_idx].directory.clone();
+            let shard_ids: Vec<u32> = shards.iter().map(|(_, sid)| *sid).collect();
+
+            if use_local_idx {
+                info!(
+                    volume_id = key.vid.0,
+                    collection = %key.collection,
+                    directory = %loc_dir,
+                    "loading orphan EC shards against locally-mirrored sidecars: {:?}",
+                    shard_names,
+                );
+                let loc = &mut self.locations[loc_idx];
+                if let Err(e) = loc.mount_ec_shards(key.vid, &key.collection, &shard_ids, "") {
+                    loc.unmount_ec_shards(key.vid, &shard_ids);
+                    warn!(
+                        volume_id = key.vid.0,
+                        directory = %loc_dir,
+                        "local-mirror shard load failed: {}",
+                        e,
+                    );
+                }
+                continue;
+            }
+
             info!(
                 volume_id = key.vid.0,
                 collection = %key.collection,
                 from = %self.locations[owner.location].directory,
-                to = %self.locations[loc_idx].directory,
+                to = %loc_dir,
                 "loading orphan EC shards using index files from sibling disk (issue #9212): {:?}",
                 shard_names,
             );
-            let shard_ids: Vec<u32> = shards.iter().map(|(_, sid)| *sid).collect();
             let owner_idx_dir = owner.idx_dir.clone();
             let loc = &mut self.locations[loc_idx];
             if let Err(e) = loc.mount_ec_shards_with_idx_dir(
@@ -141,7 +172,7 @@ impl Store {
                 loc.unmount_ec_shards(key.vid, &shard_ids);
                 warn!(
                     volume_id = key.vid.0,
-                    directory = %loc.directory,
+                    directory = %loc_dir,
                     "cross-disk shard load failed: {}",
                     e,
                 );

seaweed-volume/src/storage/volume.rs

@@ -2122,7 +2122,34 @@ impl Volume {
     }
 
     /// Mark this volume as writable (allow writes and deletes).
+    ///
+    /// If the volume booted with .vif ReadOnly=true, `load_index` built the
+    /// needle map without an .idx writer attached, so subsequent puts would
+    /// silently skip the on-disk append and only mutate in-memory state —
+    /// surviving until the next restart, then vanishing. Re-attach a writer
+    /// here so writes persist again.
     pub fn set_writable(&mut self) -> Result<(), VolumeError> {
+        // Attach the writer (if missing) before flipping the flag — otherwise
+        // a transient open/metadata failure would leave the volume marked
+        // writable with no .idx writer, and subsequent puts would silently
+        // skip the on-disk append and vanish on the next restart.
+        let needs_idx_writer = self
+            .nm
+            .as_ref()
+            .map(|nm| !nm.has_idx_writer())
+            .unwrap_or(false);
+        if needs_idx_writer {
+            let idx_path = self.file_name(".idx");
+            let write_file = OpenOptions::new()
+                .write(true)
+                .append(true)
+                .create(true)
+                .open(&idx_path)?;
+            let idx_size = write_file.metadata()?.len();
+            if let Some(ref mut nm) = self.nm {
+                nm.set_idx_file(Box::new(write_file), idx_size);
+            }
+        }
         self.no_write_or_delete = false;
         self.save_vif()
     }
@@ -3565,6 +3592,118 @@ mod tests {
         assert!(matches!(err, VolumeError::Deleted));
     }
 
+    // Guard the Rust integrity-check tombstone path against the Go regression
+    // where verifyDeletedNeedleIntegrity forwarded TombstoneFileSize into the
+    // needle-size check, mismatched against the on-disk Size=0 header, and
+    // sent every volume with a trailing deletion read-only on load. The Rust
+    // check guards its size comparison with !size.is_deleted(); this test
+    // keeps that guarantee from silently regressing.
+    #[test]
+    fn test_check_volume_data_integrity_with_deletion_tombstone() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+
+        {
+            let mut v = make_test_volume(dir);
+            for i in 1..=3 {
+                let data = format!("data {}", i);
+                let mut n = Needle {
+                    id: NeedleId(i),
+                    cookie: Cookie(i as u32),
+                    data: data.as_bytes().to_vec(),
+                    data_size: data.len() as u32,
+                    ..Needle::default()
+                };
+                v.write_needle(&mut n, true).unwrap();
+            }
+            v.delete_needle(&mut Needle {
+                id: NeedleId(2),
+                cookie: Cookie(2),
+                ..Needle::default()
+            })
+            .unwrap();
+            v.sync_to_disk().unwrap();
+        }
+
+        let v = Volume::new(
+            dir,
+            dir,
+            "",
+            VolumeId(1),
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+        assert!(
+            !v.is_no_write_or_delete(),
+            "volume should not be read-only after reload with trailing deletion tombstone"
+        );
+    }
+
+    #[test]
+    fn test_scrub_empty_volume() {
+        // Mirror of Go's TestScrubVolumeData "zero-size volume without index"
+        // case (weed/storage/volume_checking_test.go): a freshly created /
+        // pre-allocated volume has a superblock-only .dat and a zero-size .idx,
+        // and must scrub clean instead of being flagged as corrupt.
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let v = make_test_volume(dir);
+
+        // .dat holds only the superblock; .idx is empty.
+        assert_eq!(v.dat_file_size().unwrap(), SUPER_BLOCK_SIZE as u64);
+
+        let (files_checked, broken) = v.scrub().unwrap();
+        assert_eq!(files_checked, 0);
+        assert!(
+            broken.is_empty(),
+            "empty volume should scrub clean, got {:?}",
+            broken
+        );
+
+        // The index-only mode must agree.
+        let (idx_checked, idx_broken) = v.scrub_index().unwrap();
+        assert_eq!(idx_checked, 0);
+        assert!(
+            idx_broken.is_empty(),
+            "empty volume should scrub_index clean, got {:?}",
+            idx_broken
+        );
+    }
+
+    #[test]
+    fn test_scrub_healthy_volume() {
+        // Mirror of Go's TestScrubVolumeData "healthy volume" case: a volume
+        // with live needles scrubs clean and the .dat size accounting matches.
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+        let mut v = make_test_volume(dir);
+
+        for i in 1..=5 {
+            let data = format!("needle data {}", i);
+            let mut n = Needle {
+                id: NeedleId(i),
+                cookie: Cookie(i as u32),
+                data: data.as_bytes().to_vec(),
+                data_size: data.len() as u32,
+                ..Needle::default()
+            };
+            v.write_needle(&mut n, true).unwrap();
+        }
+        v.sync_to_disk().unwrap();
+
+        let (files_checked, broken) = v.scrub().unwrap();
+        assert_eq!(files_checked, 5);
+        assert!(
+            broken.is_empty(),
+            "healthy volume should scrub clean, got {:?}",
+            broken
+        );
+    }
+
     #[test]
     fn test_volume_multiple_needles() {
         let tmp = TempDir::new().unwrap();
@@ -4191,6 +4330,91 @@ mod tests {
         assert!(v.no_write_can_delete);
     }
 
+    // A volume booted with .vif ReadOnly=true used to come back stuck —
+    // load_index_inmemory built the CompactNeedleMap without an .idx writer
+    // attached, and set_writable only flipped the flag and rewrote .vif.
+    // The next put silently skipped the .idx append, so the write landed in
+    // memory only and was lost on the next restart.
+    #[test]
+    fn test_set_writable_reattaches_idx_writer_after_persisted_readonly() {
+        let tmp = TempDir::new().unwrap();
+        let dir = tmp.path().to_str().unwrap();
+
+        {
+            let mut v = make_test_volume(dir);
+            let mut n = Needle {
+                id: NeedleId(1),
+                cookie: Cookie(1),
+                data: b"initial".to_vec(),
+                data_size: 7,
+                ..Needle::default()
+            };
+            v.write_needle(&mut n, true).unwrap();
+            v.set_read_only_persist(true).unwrap();
+            v.sync_to_disk().unwrap();
+        }
+
+        let mut v = Volume::new(
+            dir,
+            dir,
+            "",
+            VolumeId(1),
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+        assert!(
+            v.no_write_or_delete,
+            "reloaded volume should be read-only from .vif"
+        );
+        assert!(
+            !v.nm.as_ref().unwrap().has_idx_writer(),
+            "read-only load should not attach an .idx writer"
+        );
+
+        v.set_writable().unwrap();
+        assert!(!v.is_read_only());
+        assert!(
+            v.nm.as_ref().unwrap().has_idx_writer(),
+            "set_writable must reattach the .idx writer or post-restart writes vanish"
+        );
+
+        let mut n = Needle {
+            id: NeedleId(2),
+            cookie: Cookie(2),
+            data: b"after-mark-writable".to_vec(),
+            data_size: 19,
+            ..Needle::default()
+        };
+        v.write_needle(&mut n, true).unwrap();
+        v.sync_to_disk().unwrap();
+
+        // Reload one more time — the .idx must contain the post-mark-writable
+        // entry, not just have it in memory.
+        drop(v);
+        let v = Volume::new(
+            dir,
+            dir,
+            "",
+            VolumeId(1),
+            NeedleMapKind::InMemory,
+            None,
+            None,
+            0,
+            Version::current(),
+        )
+        .unwrap();
+        let mut probe = Needle {
+            id: NeedleId(2),
+            ..Needle::default()
+        };
+        v.read_needle(&mut probe).unwrap();
+        assert_eq!(std::str::from_utf8(&probe.data).unwrap(), "after-mark-writable");
+    }
+
     #[test]
     fn test_load_vif_defaults_local_version_and_bytes_offset() {
         let tmp = TempDir::new().unwrap();

test/erasure_coding/admin_dockertest/ec_integration_test.go

@@ -247,7 +247,7 @@ func TestEcEndToEnd(t *testing.T) {
 		"job_type": "erasure_coding",
 		"admin_runtime": map[string]interface{}{
 			"enabled":                          true,
-			"detection_interval_seconds":       1,
+			"detection_interval_minutes":       1,
 			"global_execution_concurrency":     4,
 			"per_worker_execution_concurrency": 4,
 			"max_jobs_per_detection":           100,

test/erasure_coding/multidisk_shardloss_test.go

@@ -0,0 +1,265 @@
+package erasure_coding
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/shell"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+)
+
+// TestMultiDiskECBalanceNoShardLoss is the end-to-end regression for issue 9593.
+// It runs a real cluster of multi-disk volume servers (3 servers x 4 disks),
+// EC-encodes a volume, then runs ec.balance, asserting hard invariants the older
+// integration tests only logged:
+//
+//   - after encode the full set of 14 EC shards exists,
+//   - ec.balance never loses a shard (still 14 distinct shards afterwards),
+//   - shards end up spread across more than one disk per node, and
+//   - cluster.status counts physical disks (not one per node) and matches the
+//     real on-disk distribution.
+func TestMultiDiskECBalanceNoShardLoss(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping multi-disk EC integration test in short mode")
+	}
+
+	testDir := t.TempDir()
+	ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second)
+	defer cancel()
+
+	cluster, err := startMultiDiskCluster(ctx, testDir)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	require.NoError(t, waitForServer("127.0.0.1:9334", 30*time.Second))
+	for i := 0; i < 3; i++ {
+		require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:809%d", i), 30*time.Second))
+	}
+	t.Log("waiting for multi-disk volume servers to register...")
+	time.Sleep(10 * time.Second)
+
+	commandEnv := shell.NewCommandEnv(&shell.ShellOptions{
+		Masters:        stringPtr("127.0.0.1:9334"),
+		GrpcDialOption: grpc.WithInsecure(),
+		FilerGroup:     stringPtr("default"),
+	})
+	connectToMasterAndSync(ctx, t, commandEnv)
+
+	// Upload enough small files that the volume holds real data to encode.
+	var volumeId needle.VolumeId
+	for retry := 0; retry < 5; retry++ {
+		volumeId, err = uploadTestDataToMaster([]byte(strings.Repeat("multidisk-ec-9593 ", 64)), "127.0.0.1:9334")
+		if err == nil {
+			break
+		}
+		time.Sleep(3 * time.Second)
+	}
+	require.NoError(t, err, "failed to upload test data")
+	for i := 0; i < 40; i++ {
+		if _, e := uploadTestDataToMaster([]byte(strings.Repeat("filler ", 128)), "127.0.0.1:9334"); e != nil {
+			break
+		}
+	}
+	t.Logf("using volume %d", volumeId)
+	time.Sleep(3 * time.Second)
+
+	// Populate every server's disks with volumes so the encode can see and target
+	// each physical disk. The master only enumerates disks that already hold a
+	// volume or EC shard — an empty disk leaves no trace in the topology (heartbeats
+	// aggregate capacity per disk type, not per physical disk). ec.encode therefore
+	// spreads a volume's shards only across the disks the master already knows hold
+	// data on each node; if a node's data sits on a single disk, all its shards land
+	// there and ec.balance cannot redistribute them (it has no within-node
+	// cross-disk move). So spreading must be set up before encoding.
+	//
+	// volume.grow only tops up toward a writable target and stops on the first
+	// allocation error, so a single -count grow can create far fewer volumes than
+	// asked and leave a node on one disk. Grow repeatedly on the nodes that have not
+	// spread yet (the volume server places each new volume on its least-loaded disk)
+	// until the master's topology shows every node holding volumes on at least two
+	// physical disks. This makes the multi-disk layout — and thus the post-encode
+	// disk spread — deterministic instead of racing volume-growth and heartbeat.
+	require.Eventually(t, func() bool {
+		spread := nodeVolumeDiskCounts(t, commandEnv)
+		if len(spread) == 3 && allAtLeast(spread, 2) {
+			return true
+		}
+		for i := 0; i < 3; i++ {
+			server := fmt.Sprintf("127.0.0.1:809%d", i)
+			if spread[server] < 2 {
+				captureCommandOutput(t, shell.Commands[findCommandIndex("volume.grow")],
+					[]string{"-collection", "test", "-dataNode", server, "-count", "4"}, commandEnv)
+			}
+		}
+		return false
+	}, 60*time.Second, 2*time.Second,
+		"volumes never spread across >=2 disks on all 3 nodes")
+
+	locked, unlock := tryLockWithTimeout(t, commandEnv, 15*time.Second)
+	require.True(t, locked, "could not acquire shell lock")
+	defer unlock()
+
+	// EC-encode the volume.
+	out, err := captureCommandOutput(t, shell.Commands[findCommandIndex("ec.encode")],
+		[]string{"-volumeId", fmt.Sprintf("%d", volumeId), "-collection", "test", "-force"}, commandEnv)
+	t.Logf("ec.encode output:\n%s", out)
+	require.NoError(t, err, "ec.encode failed")
+
+	// All 14 shards must exist after encoding.
+	require.Eventually(t, func() bool {
+		return len(collectDistinctShardIDs(testDir, uint32(volumeId))) == erasureShardCount
+	}, 30*time.Second, time.Second, "expected all %d EC shards after encode, got %v",
+		erasureShardCount, collectDistinctShardIDs(testDir, uint32(volumeId)))
+
+	beforeBalance := collectDistinctShardIDs(testDir, uint32(volumeId))
+	t.Logf("after encode: %d distinct shards on %d disks", len(beforeBalance), disksWithShards(testDir, uint32(volumeId)))
+
+	// Run ec.balance.
+	out, err = captureCommandOutput(t, shell.Commands[findCommandIndex("ec.balance")],
+		[]string{"-collection", "test", "-force"}, commandEnv)
+	t.Logf("ec.balance output:\n%s", out)
+	require.NoError(t, err, "ec.balance failed")
+	time.Sleep(3 * time.Second)
+
+	// The core regression: ec.balance must not lose any shard.
+	afterBalance := collectDistinctShardIDs(testDir, uint32(volumeId))
+	require.Equal(t, erasureShardCount, len(afterBalance),
+		"ec.balance lost shards on multi-disk nodes: had %v, now %v", sortedKeysOf(beforeBalance), sortedKeysOf(afterBalance))
+
+	// Shards must be spread across more than one physical disk per node overall.
+	usedDisks := disksWithShards(testDir, uint32(volumeId))
+	assert.Greater(t, usedDisks, 3, "EC shards should span more than one disk per node (got %d disks across 3 nodes)", usedDisks)
+
+	// cluster.status must count physical disks, not collapse to one per node: it
+	// must report at least the disks actually holding this volume's shards (which
+	// is already >3 across the 3 nodes). Before the fix it reported 3 (node count).
+	require.Eventually(t, func() bool {
+		n, ok := clusterStatusDiskCount(t, commandEnv)
+		return ok && n >= usedDisks
+	}, 30*time.Second, 2*time.Second, "cluster.status never reported the >=%d physical disks holding shards (multi-disk count)", usedDisks)
+
+	n, _ := clusterStatusDiskCount(t, commandEnv)
+	t.Logf("cluster.status reports %d physical disks (>= %d holding this volume's shards)", n, usedDisks)
+}
+
+const erasureShardCount = 14 // 10 data + 4 parity
+
+// collectDistinctShardIDs returns the set of EC shard ids present for a volume
+// across every disk of every server in the multi-disk test layout.
+func collectDistinctShardIDs(testDir string, volumeId uint32) map[int]bool {
+	ids := map[int]bool{}
+	for server := 0; server < 3; server++ {
+		for disk := 0; disk < 4; disk++ {
+			diskDir := filepath.Join(testDir, fmt.Sprintf("server%d_disk%d", server, disk))
+			files, err := listECShardFiles(diskDir, volumeId)
+			if err != nil {
+				continue
+			}
+			for _, f := range files {
+				i := strings.LastIndex(f, ".ec")
+				if i < 0 {
+					continue
+				}
+				if n, err := strconv.Atoi(f[i+3:]); err == nil && n >= 0 && n < erasureShardCount {
+					ids[n] = true
+				}
+			}
+		}
+	}
+	return ids
+}
+
+// disksWithShards counts how many physical disks hold at least one shard.
+func disksWithShards(testDir string, volumeId uint32) int {
+	n := 0
+	for _, disks := range countShardsPerDisk(testDir, volumeId) {
+		for _, c := range disks {
+			if c > 0 {
+				n++
+			}
+		}
+	}
+	return n
+}
+
+// nodeVolumeDiskCounts returns, per volume server id, how many distinct physical
+// disks hold at least one volume according to the master's topology. The master
+// only enumerates disks that already hold a volume or EC shard (heartbeats
+// aggregate capacity per disk type, not per physical disk), so this reports the
+// disks ec.encode can actually spread a volume's shards across on each node.
+func nodeVolumeDiskCounts(t *testing.T, commandEnv *shell.CommandEnv) map[string]int {
+	t.Helper()
+	var resp *master_pb.VolumeListResponse
+	err := commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
+		var e error
+		resp, e = client.VolumeList(context.Background(), &master_pb.VolumeListRequest{})
+		return e
+	})
+	counts := map[string]int{}
+	if err != nil || resp.GetTopologyInfo() == nil {
+		return counts
+	}
+	for _, dc := range resp.GetTopologyInfo().GetDataCenterInfos() {
+		for _, r := range dc.GetRackInfos() {
+			for _, dn := range r.GetDataNodeInfos() {
+				disks := map[uint32]bool{}
+				for _, di := range dn.GetDiskInfos() {
+					for _, vi := range di.GetVolumeInfos() {
+						disks[vi.GetDiskId()] = true
+					}
+				}
+				counts[dn.Id] = len(disks)
+			}
+		}
+	}
+	return counts
+}
+
+func allAtLeast(counts map[string]int, min int) bool {
+	for _, c := range counts {
+		if c < min {
+			return false
+		}
+	}
+	return true
+}
+
+var diskCountRe = regexp.MustCompile(`(\d+)\s+disks?`)
+
+// clusterStatusDiskCount runs cluster.status and parses the reported disk count.
+func clusterStatusDiskCount(t *testing.T, commandEnv *shell.CommandEnv) (int, bool) {
+	t.Helper()
+	out, err := captureCommandOutput(t, shell.Commands[findCommandIndex("cluster.status")], []string{}, commandEnv)
+	if err != nil {
+		return 0, false
+	}
+	m := diskCountRe.FindStringSubmatch(out)
+	if m == nil {
+		return 0, false
+	}
+	n, err := strconv.Atoi(m[1])
+	return n, err == nil
+}
+
+func sortedKeysOf(m map[int]bool) []int {
+	out := make([]int, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	for i := 1; i < len(out); i++ {
+		for j := i; j > 0 && out[j-1] > out[j]; j-- {
+			out[j-1], out[j] = out[j], out[j-1]
+		}
+	}
+	return out
+}

test/fuse_dlm/posix_lock_ring_test.go

@@ -0,0 +1,232 @@
+package fuse_dlm
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/require"
+)
+
+// posixLockKey returns the routed-lock key a mount uses for a file at the mount
+// root. Mounts run with -filer.path=/, so the file's filer path is "/"+name; the
+// prefix must match mount.posixLockKeyForInode ("s3.fuse.lock:").
+func posixLockKey(name string) string { return "s3.fuse.lock:/" + name }
+
+// These tests exercise cross-mount POSIX advisory locks (flock), which the
+// mounts route to the inode's owner filer because they run with -dlm
+// (crossMountLocks() == lockClient != nil). They reuse the dlmTestCluster:
+// master + volume + 2 filers (forming the lock ring) + 2 mounts on filer0.
+//
+// All flock opens are O_RDONLY on purpose: a write open (O_RDWR/O_WRONLY) would
+// also take the -dlm whole-file write lock (held until close), which is a
+// different mechanism — opening read-only isolates the POSIX advisory flock.
+
+// requireForwardedLocks skips on platforms where the kernel does not forward
+// advisory locks to the FUSE server. Only Linux forwards flock/fcntl (SETLK) to
+// the filesystem; macFUSE handles flock in-kernel per mount, so cross-mount
+// coordination can't be observed there even though the routed path is correct.
+func requireForwardedLocks(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skipf("advisory locks are only forwarded to the FUSE server on Linux (GOOS=%s)", runtime.GOOS)
+	}
+}
+
+// openFlock opens path read-only and takes a flock of type how (e.g. LOCK_EX).
+// The file must already exist. The returned file holds the lock until closed.
+func openFlock(path string, how int) (*os.File, error) {
+	f, err := os.OpenFile(path, os.O_RDONLY, 0)
+	if err != nil {
+		return nil, err
+	}
+	if err := syscall.Flock(int(f.Fd()), how); err != nil {
+		f.Close()
+		return nil, err
+	}
+	return f, nil
+}
+
+// createFile creates an empty file; the brief -dlm write lock it takes is
+// released on close, before any flock test runs.
+func createFile(t *testing.T, path string) {
+	t.Helper()
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	require.NoError(t, err, "create %s", path)
+	require.NoError(t, f.Close())
+}
+
+// waitVisible waits for path to appear (cross-mount metadata propagation).
+func waitVisible(t *testing.T, path string) {
+	t.Helper()
+	require.Eventually(t, func() bool {
+		_, err := os.Stat(path)
+		return err == nil
+	}, 15*time.Second, 200*time.Millisecond, "%s never became visible", path)
+}
+
+// tryExclusiveFlock attempts a non-blocking exclusive flock on path and
+// classifies the outcome: acquired (and released again), blocked by another
+// owner (EWOULDBLOCK/EAGAIN), or an unexpected error. It distinguishes a real
+// "held by another" from incidental errors so the latter can't masquerade as a
+// satisfied "must be blocked" assertion.
+func tryExclusiveFlock(path string) (acquired, blocked bool, err error) {
+	f, err := os.OpenFile(path, os.O_RDONLY, 0)
+	if err != nil {
+		return false, false, err
+	}
+	defer f.Close()
+	if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) {
+			return false, true, nil
+		}
+		return false, false, err
+	}
+	syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+	return true, false, nil
+}
+
+// requireEventuallyBlocked asserts path becomes held by another owner. It polls
+// because a routed lock RPC can hit a transient EIO (a cold or forwarded gRPC
+// call under load) even while the lock is genuinely held; it still fails if the
+// lock is never blocked — whether it can be acquired (a double-grant) or errors
+// persistently — so an incidental error can't masquerade as "held".
+func requireEventuallyBlocked(t *testing.T, path string, timeout time.Duration, msg string) {
+	t.Helper()
+	require.Eventuallyf(t, func() bool { return isBlocked(path) },
+		timeout, 500*time.Millisecond, "%s", msg)
+}
+
+// isBlocked / isAcquirable are lenient predicates for polling, where transient
+// errors during a migration are expected and simply mean "not yet".
+func isBlocked(path string) bool { _, b, _ := tryExclusiveFlock(path); return b }
+func isAcquirable(path string) bool {
+	a, _, _ := tryExclusiveFlock(path)
+	return a
+}
+
+// stopFiler stops filer idx and clears its command so cluster teardown does not
+// try to stop it again.
+func (c *dlmTestCluster) stopFiler(idx int) {
+	stopCmd(c.filerCmds[idx])
+	c.filerCmds[idx] = nil
+}
+
+// TestPosixLockCrossMount verifies a flock taken on one mount is seen by the
+// other mount of the same cluster (the routed-to-owner-filer path end to end).
+func TestPosixLockCrossMount(t *testing.T) {
+	requireForwardedLocks(t)
+	c := startDLMTestCluster(t)
+
+	name := "posix-xmount.lock"
+	path0 := filepath.Join(c.mountPoints[0], name)
+	path1 := filepath.Join(c.mountPoints[1], name)
+
+	createFile(t, path0)
+	waitVisible(t, path1)
+
+	held, err := openFlock(path0, syscall.LOCK_EX)
+	require.NoError(t, err, "mount0 should acquire the exclusive flock")
+	defer held.Close()
+
+	requireEventuallyBlocked(t, path1, 15*time.Second, "mount1 must be blocked while mount0 holds the flock")
+
+	require.NoError(t, syscall.Flock(int(held.Fd()), syscall.LOCK_UN))
+	require.Eventually(t, func() bool { return isAcquirable(path1) },
+		15*time.Second, 500*time.Millisecond,
+		"mount1 should acquire the flock after mount0 releases it")
+}
+
+// TestPosixLockSurvivesFilerLoss verifies advisory locks held across mounts
+// survive a filer leaving the ring: ownership of the affected keys migrates to
+// the surviving filer and the holding mount re-asserts them there, so the locks
+// stay honored. It locks many files so that — with 2 filers — several are owned
+// by filer1 and thus actually migrate when filer1 stops.
+func TestPosixLockSurvivesFilerLoss(t *testing.T) {
+	requireForwardedLocks(t)
+	c := startDLMTestCluster(t)
+
+	const n = 12
+
+	// Select files via the same ring the filers run so the locked set provably
+	// spans both filers: filer1-owned keys must migrate when filer1 stops, while
+	// filer0-owned keys must keep working. Choosing by ownership (instead of
+	// hoping a sequential set happens to spread) keeps the migration path
+	// exercised on every run, independent of the cluster's dynamic ports.
+	ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	ring.SetServers([]pb.ServerAddress{
+		pb.ServerAddress(c.filerAddress(0)),
+		pb.ServerAddress(c.filerAddress(1)),
+	})
+	filer1 := pb.ServerAddress(c.filerAddress(1))
+	var onFiler1, onFiler0 []string
+	for i := 0; (len(onFiler1) < n/2 || len(onFiler0) < n/2) && i < 1000; i++ {
+		name := fmt.Sprintf("posix-migrate-%d.lock", i)
+		if ring.GetPrimary(posixLockKey(name)) == filer1 {
+			onFiler1 = append(onFiler1, name)
+		} else {
+			onFiler0 = append(onFiler0, name)
+		}
+	}
+	require.GreaterOrEqualf(t, len(onFiler1), n/2, "need %d filer1-owned keys to exercise migration", n/2)
+	require.GreaterOrEqualf(t, len(onFiler0), n/2, "need %d filer0-owned keys", n/2)
+	names := append(onFiler1[:n/2:n/2], onFiler0[:n/2]...)
+
+	held := make([]*os.File, n)
+	for i, name := range names {
+		createFile(t, filepath.Join(c.mountPoints[0], name))
+		waitVisible(t, filepath.Join(c.mountPoints[1], name))
+		f, err := openFlock(filepath.Join(c.mountPoints[0], name), syscall.LOCK_EX)
+		require.NoError(t, err, "mount0 should acquire flock %s", name)
+		held[i] = f
+		defer held[i].Close()
+	}
+
+	require.Eventually(t, func() bool {
+		for _, name := range names {
+			if !isBlocked(filepath.Join(c.mountPoints[1], name)) {
+				return false
+			}
+		}
+		return true
+	}, 20*time.Second, time.Second,
+		"every lock must be held on mount1 before the ring change")
+
+	// Drop filer1 from the ring; keys it owned migrate to filer0.
+	c.stopFiler(1)
+	require.NoError(t, c.waitForFilerCount(1, 30*time.Second), "ring should drop to one filer")
+
+	// Poll until every lock is honored again on the surviving filer: the ring
+	// must propagate and the holding mount must re-assert its locks (keepalive is
+	// 5s). We assert only the settled state — the transient migration window is
+	// covered by unit tests.
+	require.Eventually(t, func() bool {
+		for _, name := range names {
+			if !isBlocked(filepath.Join(c.mountPoints[1], name)) {
+				return false
+			}
+		}
+		return true
+	}, 45*time.Second, time.Second,
+		"all locks must survive filer1 leaving the ring (migrate to filer0)")
+
+	// Releasing on mount0 frees them all: mount1 can then acquire each.
+	for _, f := range held {
+		require.NoError(t, syscall.Flock(int(f.Fd()), syscall.LOCK_UN))
+	}
+	require.Eventually(t, func() bool {
+		for _, name := range names {
+			if !isAcquirable(filepath.Join(c.mountPoints[1], name)) {
+				return false
+			}
+		}
+		return true
+	}, 20*time.Second, 500*time.Millisecond,
+		"mount1 should acquire every lock after mount0 releases them post-migration")
+}

test/fuse_integration/concurrent_operations_test.go

@@ -177,8 +177,10 @@ func testConcurrentReadWrite(t *testing.T, framework *FuseTestFramework) {
 			defer wg.Done()
 
 			for j := 0; j < 10; j++ {
-				_, err := os.ReadFile(mountPath)
-				if err != nil {
+				if err := retryTransientFUSE(func() error {
+					_, e := os.ReadFile(mountPath)
+					return e
+				}); err != nil {
 					addError(fmt.Errorf("reader %d: %v", readerID, err))
 					return
 				}
@@ -196,8 +198,9 @@ func testConcurrentReadWrite(t *testing.T, framework *FuseTestFramework) {
 
 			for j := 0; j < 5; j++ {
 				newData := bytes.Repeat([]byte(fmt.Sprintf("WRITER%d", writerID)), 1000)
-				err := os.WriteFile(mountPath, newData, 0644)
-				if err != nil {
+				if err := retryTransientFUSE(func() error {
+					return os.WriteFile(mountPath, newData, 0644)
+				}); err != nil {
 					addError(fmt.Errorf("writer %d: %v", writerID, err))
 					return
 				}
@@ -213,6 +216,21 @@ func testConcurrentReadWrite(t *testing.T, framework *FuseTestFramework) {
 	framework.AssertFileExists(filename)
 }
 
+// retryTransientFUSE retries op a few times before giving up. A concurrent
+// truncating overwrite can leave a short-lived dentry/cache window where the
+// entry is momentarily invisible (ENOENT) to another opener; the last error is
+// returned so a genuine, persistent failure still surfaces.
+func retryTransientFUSE(op func() error) error {
+	var err error
+	for attempt := 0; attempt < 5; attempt++ {
+		if err = op(); err == nil {
+			return nil
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return err
+}
+
 // testConcurrentDirectoryOperations tests concurrent directory operations
 func testConcurrentDirectoryOperations(t *testing.T, framework *FuseTestFramework) {
 	numWorkers := 8

test/plugin_workers/erasure_coding/detection_test.go

@@ -207,8 +207,10 @@ func buildVolumeListResponse(t *testing.T, spec topologySpec, volumeID uint32) *
 	t.Helper()
 
 	volumeSizeLimitMB := uint64(100)
-	volumeSize := uint64(90) * 1024 * 1024
-	volumeModifiedAt := time.Now().Add(-10 * time.Minute).Unix()
+	// Exceed the default fullness (0.95) and quiet (1h) thresholds so volumes are
+	// EC-eligible.
+	volumeSize := uint64(96) * 1024 * 1024
+	volumeModifiedAt := time.Now().Add(-2 * time.Hour).Unix()
 
 	diskTypes := spec.diskTypes
 	if len(diskTypes) == 0 {

test/plugin_workers/erasure_coding/large_topology_test.go

@@ -29,9 +29,11 @@ func TestErasureCodingDetectionLargeTopology(t *testing.T) {
 	}
 
 	nodesPerRack := serverCount / rackCount
-	eligibleSize := uint64(90) * 1024 * 1024
+	// Eligible volumes must exceed the default fullness (0.95) and quiet (1h)
+	// thresholds; ineligible ones fall below the fullness threshold.
+	eligibleSize := uint64(96) * 1024 * 1024
 	ineligibleSize := uint64(10) * 1024 * 1024
-	modifiedAt := time.Now().Add(-10 * time.Minute).Unix()
+	modifiedAt := time.Now().Add(-2 * time.Hour).Unix()
 
 	volumeID := uint32(1)
 	dataCenters := make([]*master_pb.DataCenterInfo, 0, 1)

test/plugin_workers/fake_volume_server.go

@@ -292,6 +292,20 @@ func (v *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_serv
 	return &volume_server_pb.VolumeEcShardsMountResponse{}, nil
 }
 
+// VolumeEcShardsUnmount is a no-op stub: the worker's pre-distribute
+// cleanup calls it against every destination, and the fake server has no
+// mounted state to clear.
+func (v *VolumeServer) VolumeEcShardsUnmount(ctx context.Context, req *volume_server_pb.VolumeEcShardsUnmountRequest) (*volume_server_pb.VolumeEcShardsUnmountResponse, error) {
+	return &volume_server_pb.VolumeEcShardsUnmountResponse{}, nil
+}
+
+// VolumeEcShardsDelete is a no-op stub paired with VolumeEcShardsUnmount
+// above; the fake server doesn't persist shard files beyond what
+// ReceiveFile wrote, so there's nothing to remove.
+func (v *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_server_pb.VolumeEcShardsDeleteRequest) (*volume_server_pb.VolumeEcShardsDeleteResponse, error) {
+	return &volume_server_pb.VolumeEcShardsDeleteResponse{}, nil
+}
+
 func (v *VolumeServer) VolumeEcShardsInfo(ctx context.Context, req *volume_server_pb.VolumeEcShardsInfoRequest) (*volume_server_pb.VolumeEcShardsInfoResponse, error) {
 	if req == nil {
 		return nil, fmt.Errorf("VolumeEcShardsInfo request is nil")

test/s3/iam/STS_DISTRIBUTED.md

@@ -298,6 +298,54 @@ User Request → Load Balancer → Any S3 Gateway Instance
                               Allow/Deny Request
 ```
 
+## Trust Policy Conditions
+
+Step 5 above evaluates the role's trust policy against context keys derived from the
+OIDC token's claims. The available keys are:
+
+| Condition key | Source |
+|---------------|--------|
+| `oidc:iss` | `iss` claim (issuer URL) |
+| `oidc:sub` | `sub` claim |
+| `oidc:aud` | `aud` claim |
+| `oidc:<claim>` | any other token claim, e.g. `oidc:roles`, `oidc:groups`, `oidc:email` |
+| `aws:FederatedProvider` | the provider `name` (e.g. `keycloak-oidc`) when its configured issuer matches the token, otherwise the raw issuer URL |
+| `aws:userid` | `sub` claim (same value as `oidc:sub` during trust-policy evaluation) |
+| `sts:DurationSeconds` | requested session duration, when supplied |
+
+During trust-policy evaluation `aws:userid` is the raw `sub` claim. Once the
+role has been assumed, the keys seen by request authorization differ: there
+`aws:userid` is a stable per-identity hash of `sub` and `iss` (see
+`ComputeParentUser`), so do not assume the two contexts carry the same value.
+
+Custom claims are always exposed under the `oidc:` prefix, so a trust policy must use
+`oidc:roles` (not a bare `roles`) to match a `roles` claim:
+
+```json
+"Condition": {
+  "StringEquals": {
+    "oidc:roles": "s3-admin"
+  }
+}
+```
+
+A multi-valued claim (such as a `roles` array) matches when any of its values equals
+the condition value. The same `oidc:` keys can be interpolated into policy resources,
+e.g. `arn:aws:s3:::bucket/${oidc:sub}/*`.
+
+### roleMapping vs. trust policy
+
+A provider's `roleMapping` and a role's trust policy apply to two different entry
+points and are not interchangeable:
+
+- **Direct OIDC** — an S3 request carrying `Authorization: Bearer <OIDC-JWT>`. The
+  gateway applies `roleMapping` to choose the caller's role from the token claims; the
+  first matching rule (or `defaultRole`) wins.
+- **STS `AssumeRoleWithWebIdentity`** — the caller names the role explicitly via
+  `RoleArn`, and that role's trust policy decides whether the assumption is allowed.
+  `roleMapping` does not select the role on this path; instead the token claims are
+  surfaced as the `oidc:` condition keys above for the trust policy to evaluate.
+
 ## Configuration Management
 
 ### Development Environment

test/s3/iam/iam_config_docker.json

@@ -37,7 +37,7 @@
             "Action": ["sts:AssumeRoleWithWebIdentity"],
             "Condition": {
               "StringEquals": {
-                "roles": "s3-admin"
+                "oidc:roles": "s3-admin"
               }
             }
           }
@@ -60,7 +60,7 @@
             "Action": ["sts:AssumeRoleWithWebIdentity"],
             "Condition": {
               "StringEquals": {
-                "roles": "s3-read-only"
+                "oidc:roles": "s3-read-only"
               }
             }
           }
@@ -83,7 +83,7 @@
             "Action": ["sts:AssumeRoleWithWebIdentity"],
             "Condition": {
               "StringEquals": {
-                "roles": "s3-read-write"
+                "oidc:roles": "s3-read-write"
               }
             }
           }

test/s3/iam/s3_iam_inline_policy_condition_test.go

@@ -0,0 +1,309 @@
+package iam
+
+import (
+	"fmt"
+	mathrand "math/rand"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// uniqueResourceSuffix returns a lowercased per-test, per-invocation suffix
+// safe for use in IAM resource and S3 bucket names. Avoids EntityAlreadyExists
+// / BucketAlreadyExists collisions when integration jobs retry or run in
+// parallel against a shared stack.
+func uniqueResourceSuffix(t *testing.T) string {
+	name := strings.ToLower(t.Name())
+	name = strings.ReplaceAll(name, "/", "-")
+	name = strings.ReplaceAll(name, "_", "-")
+	return fmt.Sprintf("%s-%d", name, mathrand.Intn(10000))
+}
+
+// isAccessDenied returns true when err is an AWS error with code "AccessDenied".
+// Used to gate deny-path polling so transient setup errors don't end the
+// Eventually loop prematurely.
+func isAccessDenied(err error) bool {
+	if err == nil {
+		return false
+	}
+	awsErr, ok := err.(awserr.Error)
+	return ok && awsErr.Code() == "AccessDenied"
+}
+
+// TestIAMUserInlinePolicySourceIpCondition verifies that an aws:SourceIp condition
+// on a user inline policy is honored. Tests run from localhost (127.0.0.1), so a
+// policy that only allows access from a non-loopback CIDR must deny the request,
+// and a policy that allows access from 127.0.0.0/8 must allow it.
+func TestIAMUserInlinePolicySourceIpCondition(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	suffix := uniqueResourceSuffix(t)
+	userName := "user-" + suffix
+	policyName := "policy-" + suffix
+	bucketName := "bucket-" + suffix
+
+	_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
+	require.NoError(t, err)
+
+	keyResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{
+		UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+	accessKeyId := *keyResp.AccessKey.AccessKeyId
+	secretKey := *keyResp.AccessKey.SecretAccessKey
+
+	userS3 := createS3Client(t, accessKeyId, secretKey)
+
+	adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+	require.NoError(t, framework.CreateBucketWithCleanup(adminS3, bucketName))
+
+	t.Cleanup(func() {
+		if _, err := iamClient.DeleteUserPolicy(&iam.DeleteUserPolicyInput{
+			UserName:   aws.String(userName),
+			PolicyName: aws.String(policyName),
+		}); err != nil {
+			t.Logf("cleanup: failed to delete user policy: %v", err)
+		}
+		if _, err := iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
+			UserName:    aws.String(userName),
+			AccessKeyId: keyResp.AccessKey.AccessKeyId,
+		}); err != nil {
+			t.Logf("cleanup: failed to delete access key: %v", err)
+		}
+		if _, err := iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)}); err != nil {
+			t.Logf("cleanup: failed to delete user: %v", err)
+		}
+	})
+
+	policyDoc := func(cidrs ...string) string {
+		quoted := make([]string, len(cidrs))
+		for i, c := range cidrs {
+			quoted[i] = `"` + c + `"`
+		}
+		return `{
+			"Version":"2012-10-17",
+			"Statement":[{
+				"Effect":"Allow",
+				"Action":"s3:*",
+				"Resource":["arn:aws:s3:::` + bucketName + `","arn:aws:s3:::` + bucketName + `/*"],
+				"Condition":{"IpAddress":{"aws:SourceIp":[` + strings.Join(quoted, ",") + `]}}
+			}]
+		}`
+	}
+
+	t.Run("denies_when_source_ip_does_not_match", func(t *testing.T) {
+		// SourceIp 198.51.100.0/24 is RFC5737 TEST-NET-2; the test client is on
+		// loopback (127.0.0.1 or ::1 depending on resolver), so the condition
+		// must fail and the action must be denied.
+		_, err = iamClient.PutUserPolicy(&iam.PutUserPolicyInput{
+			UserName:       aws.String(userName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(policyDoc("198.51.100.0/24")),
+		})
+		require.NoError(t, err)
+
+		var lastErr error
+		require.Eventually(t, func() bool {
+			_, lastErr = userS3.PutObject(&s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String("denied.txt"),
+				Body:   aws.ReadSeekCloser(strings.NewReader("nope")),
+			})
+			return isAccessDenied(lastErr)
+		}, 10*time.Second, 500*time.Millisecond,
+			"PutObject must be denied with AccessDenied when aws:SourceIp condition does not match (last error: %v)", lastErr)
+	})
+
+	t.Run("allows_when_source_ip_matches", func(t *testing.T) {
+		// Cover both IPv4 and IPv6 loopback: on CI runners `localhost` may
+		// resolve to ::1 first, in which case a 127.0.0.0/8-only allow would
+		// silently never match and the test would hang.
+		_, err = iamClient.PutUserPolicy(&iam.PutUserPolicyInput{
+			UserName:       aws.String(userName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(policyDoc("127.0.0.0/8", "::1/128")),
+		})
+		require.NoError(t, err)
+
+		require.Eventually(t, func() bool {
+			_, err := userS3.PutObject(&s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String("allowed.txt"),
+				Body:   aws.ReadSeekCloser(strings.NewReader("ok")),
+			})
+			return err == nil
+		}, 10*time.Second, 500*time.Millisecond,
+			"PutObject must succeed when aws:SourceIp condition matches the loopback range")
+	})
+}
+
+// TestIAMGroupInlinePolicyEnforcement verifies that PutGroupPolicy is supported
+// and that the resulting inline policy is enforced for members of the group,
+// including its Condition block.
+func TestIAMGroupInlinePolicyEnforcement(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	suffix := uniqueResourceSuffix(t)
+	groupName := "group-" + suffix
+	userName := "user-" + suffix
+	policyName := "policy-" + suffix
+	bucketName := "bucket-" + suffix
+
+	_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
+	require.NoError(t, err)
+
+	keyResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{
+		UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+
+	_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(groupName)})
+	require.NoError(t, err)
+
+	_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
+		GroupName: aws.String(groupName),
+		UserName:  aws.String(userName),
+	})
+	require.NoError(t, err)
+
+	userS3 := createS3Client(t, *keyResp.AccessKey.AccessKeyId, *keyResp.AccessKey.SecretAccessKey)
+
+	adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+	require.NoError(t, framework.CreateBucketWithCleanup(adminS3, bucketName))
+
+	t.Cleanup(func() {
+		if _, err := iamClient.DeleteGroupPolicy(&iam.DeleteGroupPolicyInput{
+			GroupName:  aws.String(groupName),
+			PolicyName: aws.String(policyName),
+		}); err != nil {
+			t.Logf("cleanup: failed to delete group policy: %v", err)
+		}
+		if _, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
+			GroupName: aws.String(groupName),
+			UserName:  aws.String(userName),
+		}); err != nil {
+			t.Logf("cleanup: failed to remove user from group: %v", err)
+		}
+		if _, err := iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
+			UserName:    aws.String(userName),
+			AccessKeyId: keyResp.AccessKey.AccessKeyId,
+		}); err != nil {
+			t.Logf("cleanup: failed to delete access key: %v", err)
+		}
+		if _, err := iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)}); err != nil {
+			t.Logf("cleanup: failed to delete user: %v", err)
+		}
+		if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
+			t.Logf("cleanup: failed to delete group: %v", err)
+		}
+	})
+
+	// Cover both IPv4 and IPv6 loopback in the allow CIDR list: on CI runners
+	// `localhost` may resolve to ::1 first, in which case a 127.0.0.0/8-only
+	// allow would silently never match and the test would hang.
+	allowDoc := `{
+		"Version":"2012-10-17",
+		"Statement":[{
+			"Effect":"Allow",
+			"Action":"s3:*",
+			"Resource":["arn:aws:s3:::` + bucketName + `","arn:aws:s3:::` + bucketName + `/*"],
+			"Condition":{"IpAddress":{"aws:SourceIp":["127.0.0.0/8","::1/128"]}}
+		}]
+	}`
+	denyDoc := `{
+		"Version":"2012-10-17",
+		"Statement":[{
+			"Effect":"Allow",
+			"Action":"s3:*",
+			"Resource":["arn:aws:s3:::` + bucketName + `","arn:aws:s3:::` + bucketName + `/*"],
+			"Condition":{"IpAddress":{"aws:SourceIp":"198.51.100.0/24"}}
+		}]
+	}`
+
+	t.Run("crud_round_trip", func(t *testing.T) {
+		_, err := iamClient.PutGroupPolicy(&iam.PutGroupPolicyInput{
+			GroupName:      aws.String(groupName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(allowDoc),
+		})
+		require.NoError(t, err, "PutGroupPolicy must succeed (no longer NotImplemented)")
+
+		listResp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		found := false
+		for _, name := range listResp.PolicyNames {
+			if name != nil && *name == policyName {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, "ListGroupPolicies must return the freshly added policy")
+
+		getResp, err := iamClient.GetGroupPolicy(&iam.GetGroupPolicyInput{
+			GroupName:  aws.String(groupName),
+			PolicyName: aws.String(policyName),
+		})
+		require.NoError(t, err)
+		require.NotNil(t, getResp.PolicyDocument)
+		assert.Contains(t, *getResp.PolicyDocument, "aws:SourceIp",
+			"GetGroupPolicy must round-trip the Condition block")
+	})
+
+	t.Run("enforces_allow_when_condition_matches", func(t *testing.T) {
+		_, err := iamClient.PutGroupPolicy(&iam.PutGroupPolicyInput{
+			GroupName:      aws.String(groupName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(allowDoc),
+		})
+		require.NoError(t, err)
+
+		require.Eventually(t, func() bool {
+			_, err := userS3.PutObject(&s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String("group-allowed.txt"),
+				Body:   aws.ReadSeekCloser(strings.NewReader("ok")),
+			})
+			return err == nil
+		}, 10*time.Second, 500*time.Millisecond,
+			"group member must be allowed when the group policy condition matches")
+	})
+
+	t.Run("enforces_deny_when_condition_does_not_match", func(t *testing.T) {
+		_, err := iamClient.PutGroupPolicy(&iam.PutGroupPolicyInput{
+			GroupName:      aws.String(groupName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(denyDoc),
+		})
+		require.NoError(t, err)
+
+		var lastErr error
+		require.Eventually(t, func() bool {
+			_, lastErr = userS3.PutObject(&s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String("group-denied.txt"),
+				Body:   aws.ReadSeekCloser(strings.NewReader("nope")),
+			})
+			return isAccessDenied(lastErr)
+		}, 10*time.Second, 500*time.Millisecond,
+			"group member must be denied with AccessDenied when the group policy condition does not match (last error: %v)", lastErr)
+	})
+}

test/s3/lifecycle/Makefile

@@ -9,6 +9,10 @@ S3_PORT := 8333
 MASTER_PORT := 9333
 VOLUME_PORT := 8080
 FILER_PORT := 8888
+# Pin the filer gRPC port off the FILER_PORT+10000 convention so any
+# code path that assumes the offset breaks here, not in production.
+FILER_GRPC_PORT := 18890
+ADMIN_PORT := 23646
 ACCESS_KEY ?= some_access_key1
 SECRET_KEY ?= some_secret_key1
 TEST_TIMEOUT := 10m
@@ -39,6 +43,7 @@ start-server: build-weed
 	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \
 		-dir=$(SERVER_DIR) \
 		-s3.port=$(S3_PORT) \
+		-filer.port.grpc=$(FILER_GRPC_PORT) \
 		> weed-test.log 2>&1 & \
 		echo $$! > weed-server.pid
 	@for i in $$(seq 1 90); do \
@@ -68,7 +73,8 @@ test:
 	S3_ENDPOINT=http://localhost:$(S3_PORT) \
 	S3_GRPC_ENDPOINT=localhost:$$(($(S3_PORT) + 10000)) \
 	MASTER_ENDPOINT=http://localhost:$(MASTER_PORT) \
-	FILER_GRPC_ADDRESS=localhost:$$(($(FILER_PORT) + 10000)) \
+	FILER_GRPC_ADDRESS=localhost:$(FILER_GRPC_PORT) \
+	ADMIN_ENDPOINT=http://localhost:$(ADMIN_PORT) \
 	go test -v -timeout $(TEST_TIMEOUT) -run $(TEST_PATTERN)
 
 test-with-server: start-server

test/s3/lifecycle/s3_lifecycle_admin_dispatch_test.go

@@ -0,0 +1,113 @@
+package lifecycle
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+// Drives the real admin->worker dispatch path via the run endpoint and
+// asserts the dispatch reaches the filer (no dial error) and deletes
+// the backdated object. The Makefile pins filer.port.grpc off the
+// FILER_PORT+10000 convention so a raw-address forwarding regression
+// resurfaces here.
+func TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort(t *testing.T) {
+	adminEndpoint := envOr("ADMIN_ENDPOINT", defaultAdminEndpoint)
+
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("admin-dispatch")
+	mustCreateBucket(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "expire/", 1)
+	const oldKey = "expire/old.txt"
+	putObject(t, c, bucket, oldKey, "old")
+	backdateMtime(t, fc, bucket, oldKey, 30)
+
+	waitForLifecycleWorkerReady(t, adminEndpoint)
+
+	// Lifecycle is a long-running batch; the run endpoint cancels it at
+	// this timeout, which converts a healthy run into canceled_count=1.
+	const runTimeoutSeconds = 30
+	body, err := json.Marshal(map[string]any{
+		"timeout_seconds": runTimeoutSeconds,
+	})
+	require.NoError(t, err)
+	req, err := http.NewRequestWithContext(
+		context.Background(), http.MethodPost,
+		adminEndpoint+"/api/plugin/job-types/s3_lifecycle/run",
+		bytes.NewReader(body),
+	)
+	require.NoError(t, err)
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	var payload map[string]any
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&payload))
+	t.Logf("admin /api/plugin/job-types/s3_lifecycle/run response: %v", payload)
+	require.Equal(t, http.StatusOK, resp.StatusCode, "admin run endpoint failed: %v", payload)
+
+	require.GreaterOrEqual(t, jsonNumber(t, payload, "detected_count"), 1)
+	require.Equal(t, 0, jsonNumber(t, payload, "error_count"),
+		"dispatched job errored — likely filer_grpc_address was raw host:httpPort.grpcPort")
+
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(oldKey),
+		})
+		return err != nil
+	}, 30*time.Second, 500*time.Millisecond,
+		"expected %s/%s to be deleted after admin-dispatched lifecycle run", bucket, oldKey)
+}
+
+func jsonNumber(t *testing.T, payload map[string]any, key string) int {
+	t.Helper()
+	raw, ok := payload[key]
+	require.Truef(t, ok, "response missing key %q: %v", key, payload)
+	switch v := raw.(type) {
+	case float64:
+		return int(v)
+	case json.Number:
+		n, err := v.Int64()
+		require.NoErrorf(t, err, "key %q is not an int: %v", key, raw)
+		return int(n)
+	default:
+		t.Fatalf("key %q has unexpected type %T (%v)", key, raw, raw)
+		return 0
+	}
+}
+
+func waitForLifecycleWorkerReady(t *testing.T, adminEndpoint string) {
+	t.Helper()
+	deadline := time.Now().Add(30 * time.Second)
+	for time.Now().Before(deadline) {
+		req, err := http.NewRequest(http.MethodGet, adminEndpoint+"/api/plugin/scheduler-states", nil)
+		require.NoError(t, err)
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			time.Sleep(250 * time.Millisecond)
+			continue
+		}
+		var payload any
+		_ = json.NewDecoder(resp.Body).Decode(&payload)
+		resp.Body.Close()
+		if resp.StatusCode == http.StatusOK && strings.Contains(fmt.Sprintf("%v", payload), "s3_lifecycle") {
+			return
+		}
+		time.Sleep(250 * time.Millisecond)
+	}
+	t.Fatalf("admin never reported an s3_lifecycle-capable worker")
+}

test/s3/lifecycle/s3_lifecycle_test.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -40,12 +41,14 @@ const (
 	defaultS3Endpoint     = "http://localhost:8333"
 	defaultS3GrpcEndpoint = "localhost:18333"
 	defaultMasterEndpt    = "http://localhost:9333"
-	defaultFilerGRPC      = "localhost:18888"
+	defaultAdminEndpoint  = "http://localhost:23646"
+	// Pinned off the FILER_PORT+10000 convention; see Makefile.
+	defaultFilerGRPC      = "localhost:18890"
 	bucketLifecycleXMLKey = "s3-bucket-lifecycle-configuration-xml"
 	bucketsPath           = "/buckets"
 	accessKey             = "some_access_key1"
-	secretKey           = "some_secret_key1"
-	region              = "us-east-1"
+	secretKey             = "some_secret_key1"
+	region                = "us-east-1"
 )
 
 func envOr(key, def string) string {
@@ -67,7 +70,111 @@ func s3Client(t *testing.T) *s3.Client {
 			})),
 	)
 	require.NoError(t, err)
-	return s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true })
+	client := s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true })
+	ensureClusterWritable(t, client)
+	return client
+}
+
+var clusterWritableOnce sync.Once
+
+// ensureClusterWritable blocks until the cluster can actually serve a write,
+// absorbing the volume-growth warmup window after a fresh start. The Makefile
+// only waits for the server process to be up ("server up after N s"); it does
+// not wait for a writable volume, so the first PutObject can race volume growth
+// and fail with a transient 500 (assign volume: DeadlineExceeded) — the source
+// of the lifecycle-test flakes. Probing one throwaway write here, once per
+// process, warms growth so every test's first real write is past that window.
+// Best-effort: if it never becomes writable, the test's own PutObject surfaces
+// the failure normally.
+func ensureClusterWritable(t *testing.T, c *s3.Client) {
+	t.Helper()
+	clusterWritableOnce.Do(func() {
+		bucket := uniqueBucket("warmup")
+		deadline := time.Now().Add(60 * time.Second)
+
+		// try runs fn under a bounded context so a single hung call can't block.
+		try := func(timeout time.Duration, fn func(ctx context.Context) error) error {
+			ctx, cancel := context.WithTimeout(context.Background(), timeout)
+			defer cancel()
+			return fn(ctx)
+		}
+		// probe is a try whose timeout is clamped to the time left before the
+		// deadline, so the whole warmup stays within the budget; ok=false means
+		// the budget is exhausted and the caller should stop.
+		probe := func(fn func(ctx context.Context) error) (err error, ok bool) {
+			remaining := time.Until(deadline)
+			if remaining <= 0 {
+				return nil, false
+			}
+			if remaining > 10*time.Second {
+				remaining = 10 * time.Second
+			}
+			return try(remaining, fn), true
+		}
+		// backoff sleeps attempt*250ms, never past the deadline.
+		backoff := func(attempt int) {
+			d := time.Duration(attempt) * 250 * time.Millisecond
+			if left := time.Until(deadline); d > left {
+				d = left
+			}
+			if d > 0 {
+				time.Sleep(d)
+			}
+		}
+
+		// CreateBucket is a metadata op, but on a cold cluster the filer itself may
+		// not be ready yet, so retry it within the deadline rather than abandoning
+		// the whole warmup (and the PutObject probe) on the first error.
+		created := false
+		for attempt := 1; ; attempt++ {
+			err, ok := probe(func(ctx context.Context) error {
+				_, e := c.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)})
+				return e
+			})
+			if !ok {
+				break
+			}
+			if err == nil {
+				created = true
+				break
+			}
+			backoff(attempt)
+		}
+		if !created {
+			t.Logf("warmup: could not create probe bucket within 60s; proceeding")
+			return
+		}
+		// Cleanup gets a fresh timeout (not the warmup budget) so teardown runs
+		// even when the probe loop used the full window.
+		defer try(10*time.Second, func(ctx context.Context) error {
+			_, e := c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)})
+			return e
+		})
+
+		for attempt := 1; ; attempt++ {
+			err, ok := probe(func(ctx context.Context) error {
+				_, e := c.PutObject(ctx, &s3.PutObjectInput{
+					Bucket: aws.String(bucket), Key: aws.String("warmup"), Body: strings.NewReader("ok"),
+				})
+				return e
+			})
+			if !ok {
+				break
+			}
+			if err == nil {
+				try(10*time.Second, func(ctx context.Context) error {
+					_, e := c.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String("warmup")})
+					return e
+				})
+				if attempt > 1 {
+					t.Logf("cluster became writable after %d probe(s)", attempt)
+				}
+				return
+			}
+			backoff(attempt)
+		}
+		t.Logf("warmup: cluster not confirmed writable within 60s; proceeding")
+	})
 }
 
 func filerClient(t *testing.T) (filer_pb.SeaweedFilerClient, func()) {

test/s3/sdk_v2_routing/Makefile

@@ -0,0 +1,41 @@
+# AWS SDK V2 Route Disambiguation Integration Tests
+#
+# Pins the regression for the route collision between the regular S3 API
+# and the S3 Tables REST API on shared top-level paths (/buckets,
+# /get-table). The tests use the real AWS SDK V2 for Go so the SDK's own
+# XML deserializer is the assertion — a JSON body produces an SDK error
+# before any test code runs.
+#
+# Prerequisites:
+#   - SeaweedFS running with S3 API enabled on port 8333
+#   - Go 1.21+
+#
+# Usage:
+#   make test         - Run the SDK V2 routing tests
+#   make test-verbose - Run with verbose output
+#   make clean        - Clean test cache
+
+.PHONY: all test test-verbose clean help
+
+S3_ENDPOINT ?= http://127.0.0.1:8333
+
+all: test
+
+test:
+	@echo "Running SDK V2 routing tests against $(S3_ENDPOINT)..."
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m ./...
+
+test-verbose:
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m -count=1 ./...
+
+clean:
+	go clean -testcache
+
+help:
+	@echo "AWS SDK V2 Route Disambiguation Tests"
+	@echo "Targets:"
+	@echo "  test          Run the routing tests"
+	@echo "  test-verbose  Run with verbose output"
+	@echo "  clean         Clean test cache"
+	@echo "Environment Variables:"
+	@echo "  S3_ENDPOINT   S3 endpoint URL (default: http://127.0.0.1:8333)"

test/s3/sdk_v2_routing/s3_sdk_v2_routing_test.go

@@ -0,0 +1,210 @@
+// Package sdkv2routing_test exercises route disambiguation between the
+// regular S3 API and the S3 Tables REST API on top-level paths the two
+// share (/buckets, /get-table). The bug it pins:
+//
+// When a user has an S3 bucket named "buckets" (or "get-table"), a
+// path-style ListObjectsV2 request sent by AWS SDK V2 / Hadoop s3a /
+// Spark would be routed to the S3 Tables ListTableBuckets handler and
+// receive a JSON body. AWS SDK V2 then fails XML parsing with
+// "Unexpected character '{' (code 123) in prolog".
+//
+// These tests use the real AWS SDK V2 for Go, so the SDK's own
+// deserializer is the assertion: if the server returns the wrong
+// content type, the SDK errors out before any test assertion runs.
+package sdkv2routing_test
+
+import (
+	"context"
+	"io"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	defaultEndpoint  = "http://127.0.0.1:8333"
+	defaultAccessKey = "some_access_key1"
+	defaultSecretKey = "some_secret_key1"
+	defaultRegion    = "us-east-1"
+)
+
+func getS3Client(t *testing.T) *s3.Client {
+	t.Helper()
+
+	endpoint := os.Getenv("S3_ENDPOINT")
+	if endpoint == "" {
+		endpoint = defaultEndpoint
+	}
+	accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
+	if accessKey == "" {
+		accessKey = defaultAccessKey
+	}
+	secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
+	if secretKey == "" {
+		secretKey = defaultSecretKey
+	}
+	region := os.Getenv("AWS_REGION")
+	if region == "" {
+		region = defaultRegion
+	}
+
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithRegion(region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
+		config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
+			func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+				return aws.Endpoint{
+					URL:               endpoint,
+					SigningRegion:     defaultRegion,
+					HostnameImmutable: true,
+				}, nil
+			})),
+	)
+	require.NoError(t, err)
+
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+}
+
+// ensureBucket creates bucket if it doesn't already exist. It tolerates
+// BucketAlreadyOwnedByYou / BucketAlreadyExists so the tests are
+// idempotent across local re-runs.
+func ensureBucket(t *testing.T, ctx context.Context, client *s3.Client, bucket string) {
+	t.Helper()
+	_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)})
+	if err == nil {
+		return
+	}
+	msg := err.Error()
+	if strings.Contains(msg, "BucketAlreadyOwnedByYou") || strings.Contains(msg, "BucketAlreadyExists") {
+		return
+	}
+	t.Fatalf("CreateBucket(%q) failed: %v", bucket, err)
+}
+
+// deleteBucket best-effort cleans up a bucket and any objects in it.
+// Test does not fail if cleanup fails — the next run is idempotent.
+func deleteBucket(ctx context.Context, client *s3.Client, bucket string) {
+	paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{Bucket: aws.String(bucket)})
+	for paginator.HasMorePages() {
+		page, err := paginator.NextPage(ctx)
+		if err != nil {
+			break
+		}
+		for _, obj := range page.Contents {
+			client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: obj.Key})
+		}
+	}
+	client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)})
+}
+
+// TestListObjectsV2_OnBucketNamedBuckets is the direct reproducer for
+// issue #9559: Spark / Hadoop s3a does a ListObjectsV2 against bucket
+// "buckets" via AWS SDK V2, which fails with
+// "Could not parse XML response. ... Unexpected character '{' (code 123)
+// in prolog" when SeaweedFS routes the request to the JSON-returning
+// ListTableBuckets handler. The SDK's response deserializer is the
+// real assertion here — a JSON body produces an SDK error before we
+// reach require.NoError.
+func TestListObjectsV2_OnBucketNamedBuckets(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	client := getS3Client(t)
+
+	const bucket = "buckets"
+	ensureBucket(t, ctx, client, bucket)
+	t.Cleanup(func() { deleteBucket(ctx, client, bucket) })
+
+	out, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String("logs/"),
+	})
+	require.NoError(t, err, "AWS SDK V2 must parse the response as XML")
+	require.NotNil(t, out)
+}
+
+// TestPutGetObject_OnBucketNamedBuckets exercises the full read/write
+// round-trip on the colliding bucket name. PutObject and GetObject go
+// through different routes than ListObjectsV2, and verifying them
+// guards against future regressions that re-route only some verbs.
+func TestPutGetObject_OnBucketNamedBuckets(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	client := getS3Client(t)
+
+	const bucket = "buckets"
+	const key = "logs/hello.txt"
+	const body = "hello from issue 9559"
+
+	ensureBucket(t, ctx, client, bucket)
+	t.Cleanup(func() { deleteBucket(ctx, client, bucket) })
+
+	_, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+		Body:   strings.NewReader(body),
+	})
+	require.NoError(t, err)
+
+	out, err := client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
+	require.NoError(t, err)
+	defer out.Body.Close()
+
+	got, err := io.ReadAll(out.Body)
+	require.NoError(t, err)
+	require.Equal(t, body, string(got))
+}
+
+// TestListObjectsV2_OnBucketNamedGetTable covers the second colliding
+// path. The S3 Tables GET /get-table endpoint shares its path with a
+// bucket literally named "get-table", which is a legal S3 bucket name.
+func TestListObjectsV2_OnBucketNamedGetTable(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	client := getS3Client(t)
+
+	const bucket = "get-table"
+	ensureBucket(t, ctx, client, bucket)
+	t.Cleanup(func() { deleteBucket(ctx, client, bucket) })
+
+	out, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: aws.String(bucket)})
+	require.NoError(t, err)
+	require.NotNil(t, out)
+}
+
+// TestCreateAndListBuckets_ServiceLevel verifies the SDK's service-level
+// ListBuckets still parses as XML when a bucket named "buckets" exists.
+// ListBuckets goes through GET / (root) which is unaffected by the
+// /buckets route collision — this is a guard against the matcher
+// accidentally widening to top-level paths.
+func TestCreateAndListBuckets_ServiceLevel(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	client := getS3Client(t)
+
+	const bucket = "buckets"
+	ensureBucket(t, ctx, client, bucket)
+	t.Cleanup(func() { deleteBucket(ctx, client, bucket) })
+
+	out, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
+	require.NoError(t, err)
+	require.NotNil(t, out)
+
+	found := false
+	for _, b := range out.Buckets {
+		if aws.ToString(b.Name) == bucket {
+			found = true
+			break
+		}
+	}
+	require.True(t, found, "bucket %q must appear in service-level ListBuckets", bucket)
+}
+

test/s3tables/catalog/iceberg_catalog_test.go

@@ -17,6 +17,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+
 	"github.com/seaweedfs/seaweedfs/test/testutil"
 )
 
@@ -451,19 +454,22 @@ func icebergPath(prefix, path string) string {
 	return withPrefix
 }
 
-// createTableBucket creates a table bucket via the S3Tables REST API
+// createTableBucket creates a table bucket via the S3Tables REST API.
+// The request is AWS V4 signed for SERVICE=s3tables so the S3 Tables
+// route matcher accepts it; signing with regular SERVICE=s3 would let
+// the request fall through to the S3 CreateBucket handler.
 func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
 	t.Helper()
 
-	// Use S3Tables REST API to create the bucket
 	endpoint := fmt.Sprintf("http://localhost:%d/buckets", env.s3Port)
-
 	reqBody := fmt.Sprintf(`{"name":"%s"}`, bucketName)
+
 	req, err := http.NewRequest(http.MethodPut, endpoint, strings.NewReader(reqBody))
 	if err != nil {
 		t.Fatalf("Failed to create request: %v", err)
 	}
 	req.Header.Set("Content-Type", "application/x-amz-json-1.1")
+	signS3TablesRequest(t, req, reqBody)
 
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
@@ -480,6 +486,20 @@ func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
 	t.Logf("Created table bucket %s", bucketName)
 }
 
+// signS3TablesRequest signs req with AWS V4 for SERVICE=s3tables. The
+// underlying weed mini instance runs in default-allow mode so the
+// signature itself is not verified; only the credential scope matters,
+// because the S3 Tables route matcher requires SERVICE=s3tables to
+// distinguish S3 Tables traffic from regular S3 calls on the same paths.
+func signS3TablesRequest(t *testing.T, req *http.Request, body string) {
+	t.Helper()
+	creds := credentials.NewStaticCredentials("test-ak", "test-sk", "")
+	signer := v4.NewSigner(creds)
+	if _, err := signer.Sign(req, strings.NewReader(body), "s3tables", "us-east-1", time.Now()); err != nil {
+		t.Fatalf("Failed to sign S3 Tables request: %v", err)
+	}
+}
+
 // randomSuffix returns a short random hex suffix for unique resource naming.
 func randomSuffix() string {
 	return fmt.Sprintf("%x", time.Now().UnixNano()&0xffffffff)

test/s3tables/catalog_risingwave/setup_test.go

@@ -411,7 +411,12 @@ func createIcebergTable(t *testing.T, env *TestEnvironment, bucketName, namespac
 func listFilerContents(t *testing.T, env *TestEnvironment, path string) {
 	t.Helper()
 
-	cmd := exec.Command("weed", "shell",
+	// Bound diagnostic listing so a hung weed shell during cleanup can't
+	// burn the whole 20-minute test timeout.
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, "weed", "shell",
 		fmt.Sprintf("-master=%s", env.hostMasterAddress()),
 	)
 	cmd.Stdin = strings.NewReader(fmt.Sprintf("fs.ls -R %s\nexit\n", path))

test/samba/Dockerfile

@@ -0,0 +1,20 @@
+FROM chrislusf/seaweedfs:e2e
+
+RUN apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && \
+    DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 install -y \
+    --no-install-recommends \
+    --no-install-suggests \
+    samba \
+    smbclient \
+    python3-minimal \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY smb.conf.template /smb.conf.template
+COPY smb_tests.sh /smb_tests.sh
+COPY lock_tests.sh /lock_tests.sh
+COPY entrypoint.sh /entrypoint.sh
+COPY run_inside_container.sh /run_inside_container.sh
+RUN chmod +x /smb_tests.sh /lock_tests.sh /entrypoint.sh /run_inside_container.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

test/samba/README.md

@@ -0,0 +1,96 @@
+# Samba on FUSE integration test
+
+Exports a SeaweedFS FUSE mount over SMB with Samba's `smbd` and drives it with
+`smbclient`, verifying that SMB file operations work correctly on top of the
+mount and that data stays consistent across both protocols.
+
+## What it checks
+
+The functional battery in `smb_tests.sh` covers:
+
+- connecting to the share and listing the root
+- 1 MiB upload/download round-trip with content verification
+- subdirectory creation and writes into it
+- file rename
+- 64 MiB upload/download (exercises SeaweedFS chunk splitting)
+- recursive upload of a directory tree
+- cross-protocol consistency: files written over SMB appear on the FUSE mount
+  with identical content, and files written directly on the FUSE mount are
+  readable over SMB
+- deleting files and directory trees
+
+The locking / concurrency battery in `lock_tests.sh` covers the harder cases a
+network-filesystem backend has to get right:
+
+- **POSIX `fcntl` byte-range locking** on the FUSE mount: a held exclusive lock
+  denies a conflicting lock, allows a non-overlapping range, and is reacquirable
+  after release (exercises the mount's `SetLk`/`GetLk`)
+- **Distributed locking** (`-dlm`): a file held open for writing on one mount
+  blocks a writer on a second mount until it is released
+- **Distributed-lock integrity**: concurrent writers to the same file from two
+  mounts leave exactly one intact payload, never a torn mix
+- **Concurrency**: parallel writers to distinct files all succeed
+
+Both FUSE mounts are started with `-dlm` (distributed lock manager). The second
+mount (`/mnt/seaweedfs2`) exists only to contend with the smbd-backed mount in
+the distributed-locking tests; both see the same filer path, so `.../share` is
+the same data on each.
+
+> Note on DLM semantics: `-dlm` coordinates *write access* (one mount writes a
+> file at a time) and guarantees writes are not torn. It does not guarantee
+> which concurrent writer wins or instant cross-mount read convergence — the
+> holder's buffered data is flushed on close, asynchronously to lock release.
+> When a holder closes the file, a writer on another mount acquires the freed
+> lock within ~1s and completes.
+
+## Layout
+
+| File | Purpose |
+| --- | --- |
+| `smb_tests.sh` | SMB functional battery. Shared by both runners. |
+| `lock_tests.sh` | SMB locking / concurrency battery. Shared by both runners. |
+| `smb.conf.template` | Samba config; placeholders are filled in at run time. |
+| `run.sh` | Local runner: `weed mini` + two `-dlm` mounts + `smbd` + both batteries, all as the current user on unprivileged ports. |
+| `entrypoint.sh` | Container entrypoint: starts two `-dlm` FUSE mounts and runs `smbd`. |
+| `run_inside_container.sh` | Runs both batteries inside the container against the local `smbd`. |
+| `Dockerfile` | Adds Samba to the `chrislusf/seaweedfs:e2e` image. |
+| `docker-compose.yml` | master + volume + filer + samba services. |
+
+## Running locally
+
+Requirements: `weed` on `$PATH`, `fusermount3`, and Samba's `smbd` /
+`smbclient` / `smbpasswd` (Debian/Ubuntu: `apt-get install samba smbclient`).
+
+```sh
+test/samba/run.sh
+```
+
+No `sudo` is needed: `smbd` runs as the current user on port 4450 and all state
+lives under a temp work dir that is cleaned up on exit.
+
+## Running with Docker
+
+Mirrors the CI job. Requires `/dev/fuse` and `SYS_ADMIN` (provided in the
+compose file).
+
+```sh
+# build the base e2e image first (from the repo's docker/ dir)
+docker compose -f test/samba/docker-compose.yml up --wait
+docker compose -f test/samba/docker-compose.yml exec -T samba /run_inside_container.sh
+docker compose -f test/samba/docker-compose.yml down -v
+```
+
+## CI
+
+`.github/workflows/samba-integration.yml` runs on changes to `weed/mount/**`,
+`weed/filer/**`, or `test/samba/**`. It builds the e2e image, builds the Samba
+harness image on top, brings up the cluster, runs the battery, and uploads
+server logs as artifacts.
+
+## Notes
+
+- The share disables Samba's DOS-attribute / xattr mapping and oplocks. The
+  SeaweedFS FUSE mount does not implement that surface, and leaving it on
+  produces `NT_STATUS_NOT_SUPPORTED` errors unrelated to data integrity.
+- The share path is a subdirectory of the mount (`.../share`) so the runner can
+  verify SMB-side operations directly on the FUSE side.

test/samba/docker-compose.yml

@@ -0,0 +1,60 @@
+services:
+  master:
+    image: chrislusf/seaweedfs:e2e
+    command: "-v=4 master -ip=master -ip.bind=0.0.0.0 -raftBootstrap"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz"]
+      interval: 2s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
+
+  volume:
+    image: chrislusf/seaweedfs:e2e
+    command: "-v=4 volume -master=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "-I", "http://localhost:8080/healthz"]
+      interval: 2s
+      timeout: 10s
+      retries: 15
+      start_period: 5s
+    depends_on:
+      master:
+        condition: service_healthy
+
+  filer:
+    image: chrislusf/seaweedfs:e2e
+    command: "-v=4 filer -master=master:9333 -ip=filer -ip.bind=0.0.0.0"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "-I", "http://localhost:8888/healthz"]
+      interval: 2s
+      timeout: 10s
+      retries: 15
+      start_period: 5s
+    depends_on:
+      volume:
+        condition: service_healthy
+
+  samba:
+    image: chrislusf/seaweedfs:samba
+    build:
+      context: .
+    environment:
+      FILER: filer:8888
+    cap_add:
+      - SYS_ADMIN
+    devices:
+      - /dev/fuse
+    security_opt:
+      - apparmor:unconfined
+    healthcheck:
+      test:
+        - "CMD-SHELL"
+        - "mountpoint -q /mnt/seaweedfs && mountpoint -q /mnt/seaweedfs2 && smbclient -L 127.0.0.1 -p 445 -U smbtest%smbtest -m SMB3 >/dev/null 2>&1"
+      interval: 3s
+      timeout: 10s
+      retries: 20
+      start_period: 15s
+    depends_on:
+      filer:
+        condition: service_healthy

test/samba/entrypoint.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+#
+# Entrypoint for the samba test container.
+#
+# Mounts SeaweedFS over FUSE twice, both with distributed locking (-dlm) so the
+# locking tests can exercise cross-mount write coordination:
+#   - MOUNT_DIR  (/mnt/seaweedfs)  is exported over SMB by smbd
+#   - MOUNT2_DIR (/mnt/seaweedfs2) is a second, independent mount of the same
+#     filer used to contend with the SMB writer
+#
+# Both mounts see the same filer path, so .../share is the same data on each.
+# smbd runs in the foreground (as root, which owns the mounts), so the share
+# uses "force user = root".
+set -euo pipefail
+
+FILER="${FILER:-filer:8888}"
+MOUNT_DIR="${MOUNT_DIR:-/mnt/seaweedfs}"
+MOUNT2_DIR="${MOUNT2_DIR:-/mnt/seaweedfs2}"
+SHARE_DIR="${MOUNT_DIR}/share"
+STATE_DIR="${STATE_DIR:-/var/lib/samba-test}"
+SMB_PORT="${SMB_PORT:-445}"
+SMB_USER="${SMB_USER:-smbtest}"
+SMB_PASS="${SMB_PASS:-smbtest}"
+
+mkdir -p "${MOUNT_DIR}" "${MOUNT2_DIR}" \
+  "${STATE_DIR}/private" "${STATE_DIR}/state" "${STATE_DIR}/cache" \
+  "${STATE_DIR}/lock" "${STATE_DIR}/pid" "${STATE_DIR}/ncalrpc"
+
+# mount_seaweedfs <mountpoint> <logfile> — mount with -dlm and wait for it.
+mount_seaweedfs() {
+  local dir="$1" log="$2"
+  echo "==> Mounting SeaweedFS (${FILER}) at ${dir} with -dlm"
+  weed -v=1 mount \
+    -filer="${FILER}" \
+    -dir="${dir}" \
+    -filer.path=/ \
+    -dirAutoCreate \
+    -allowOthers \
+    -dlm \
+    >"${log}" 2>&1 &
+  local pid=$!
+  for _ in $(seq 1 120); do
+    if mountpoint -q "${dir}"; then
+      return 0
+    fi
+    if ! kill -0 "${pid}" 2>/dev/null; then
+      echo "weed mount (${dir}) exited early; log tail:" >&2
+      tail -n 100 "${log}" >&2 || true
+      exit 1
+    fi
+    sleep 0.5
+  done
+  echo "FUSE mount ${dir} did not come up" >&2
+  tail -n 100 "${log}" >&2 || true
+  exit 1
+}
+
+mount_seaweedfs "${MOUNT_DIR}" /var/log/weed-mount.log
+mount_seaweedfs "${MOUNT2_DIR}" /var/log/weed-mount2.log
+
+mkdir -p "${SHARE_DIR}"
+chmod 0777 "${SHARE_DIR}"
+
+# --- configure and start smbd ----------------------------------------------
+echo "==> Configuring Samba share on port ${SMB_PORT}"
+sed -e "s#@SHARE_PATH@#${SHARE_DIR}#g" \
+  -e "s#@STATE_DIR@#${STATE_DIR}#g" \
+  -e "s#@SMB_PORT@#${SMB_PORT}#g" \
+  -e "s#@FORCE_USER@#root#g" \
+  /smb.conf.template >/etc/samba/smb.conf
+
+id -u "${SMB_USER}" >/dev/null 2>&1 || useradd -M -s /usr/sbin/nologin "${SMB_USER}"
+printf '%s\n%s\n' "${SMB_PASS}" "${SMB_PASS}" | smbpasswd -a -s "${SMB_USER}"
+
+echo "==> Starting smbd"
+exec smbd -F --no-process-group -s /etc/samba/smb.conf

test/samba/lock_tests.sh

@@ -0,0 +1,218 @@
+#!/usr/bin/env bash
+#
+# Locking / concurrency test battery for Samba on a SeaweedFS FUSE mount.
+#
+# Covers the challenges a network-filesystem backend has to get right:
+#   1. POSIX fcntl byte-range locking on the FUSE mount (SetLk/GetLk)
+#   2. Distributed locking (-dlm): a write held open on one mount blocks a
+#      writer on another mount until it is released
+#   3. Distributed locking integrity: concurrent writers to the same file from
+#      two mounts produce intact (non-torn) data
+#   4. Concurrent writers to distinct files all succeed
+#
+# Required env:
+#   SMB_USER, SMB_PASS              samba credentials
+#   MOUNT_SHARE                     dir on the smbd-backed FUSE mount (mount 1)
+#   MOUNT2_SHARE                    dir on the second FUSE mount (mount 2)
+# Optional env:
+#   SMB_HOST (127.0.0.1), SMB_SHARE (seaweedfs), SMB_PORT (445)
+set -uo pipefail
+
+SMB_HOST="${SMB_HOST:-127.0.0.1}"
+SMB_SHARE="${SMB_SHARE:-seaweedfs}"
+SMB_PORT="${SMB_PORT:-445}"
+SMB_USER="${SMB_USER:?SMB_USER is required}"
+SMB_PASS="${SMB_PASS:?SMB_PASS is required}"
+MOUNT_SHARE="${MOUNT_SHARE:?MOUNT_SHARE is required}"
+MOUNT2_SHARE="${MOUNT2_SHARE:?MOUNT2_SHARE is required}"
+
+WORK="$(mktemp -d /tmp/samba-locktest.XXXXXX)"
+trap 'rm -rf "${WORK}"' EXIT
+
+PASS=0
+FAIL=0
+pass() { printf '  [PASS] %s\n' "$1"; PASS=$((PASS + 1)); }
+fail() { printf '  [FAIL] %s\n' "$1"; FAIL=$((FAIL + 1)); }
+
+smb() {
+  smbclient "//${SMB_HOST}/${SMB_SHARE}" -p "${SMB_PORT}" \
+    -U "${SMB_USER}%${SMB_PASS}" -m SMB3 -c "$1"
+}
+md5() { md5sum "$1" | awk '{print $1}'; }
+
+# 1. POSIX fcntl byte-range locking on the FUSE mount ------------------------
+# Exercises the mount's SetLk/GetLk via two processes contending over fcntl
+# (F_SETLK) byte-range locks. python3's fcntl.lockf issues real POSIX locks.
+echo "==> 1. POSIX fcntl byte-range locking (FUSE mount SetLk/GetLk)"
+lockfile="${MOUNT_SHARE}/fcntl_lock.dat"
+: >"${lockfile}"
+fcntl_out="$(python3 - "${lockfile}" <<'PY'
+import fcntl, os, sys
+
+path = sys.argv[1]
+parent_to_child_r, parent_to_child_w = os.pipe()  # release signal
+child_to_parent_r, child_to_parent_w = os.pipe()  # locked signal
+
+pid = os.fork()
+if pid == 0:  # child: hold an exclusive lock on [0,100)
+    fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o644)
+    fcntl.lockf(fd, fcntl.LOCK_EX, 100, 0, 0)
+    os.write(child_to_parent_w, b"L")
+    os.read(parent_to_child_r, 1)  # wait until parent says release
+    fcntl.lockf(fd, fcntl.LOCK_UN, 100, 0, 0)
+    os.close(fd)
+    os._exit(0)
+
+# parent
+os.read(child_to_parent_r, 1)  # wait until child holds the lock
+fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o644)
+results = []
+
+# a. a conflicting exclusive lock must be denied while the child holds it
+try:
+    fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB, 100, 0, 0)
+    fcntl.lockf(fd, fcntl.LOCK_UN, 100, 0, 0)
+    results.append(("conflicting exclusive lock denied while held", False))
+except OSError:
+    results.append(("conflicting exclusive lock denied while held", True))
+
+# b. a non-overlapping range must be grantable
+try:
+    fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB, 100, 200, 0)
+    fcntl.lockf(fd, fcntl.LOCK_UN, 100, 200, 0)
+    results.append(("non-overlapping range lock granted", True))
+except OSError:
+    results.append(("non-overlapping range lock granted", False))
+
+# c. after the holder releases, the lock must be acquirable
+os.write(parent_to_child_w, b"R")
+os.waitpid(pid, 0)
+try:
+    fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB, 100, 0, 0)
+    fcntl.lockf(fd, fcntl.LOCK_UN, 100, 0, 0)
+    results.append(("lock acquirable after holder releases", True))
+except OSError:
+    results.append(("lock acquirable after holder releases", False))
+
+for name, ok in results:
+    print(("  [PASS] " if ok else "  [FAIL] ") + name)
+sys.exit(0 if all(ok for _, ok in results) else 1)
+PY
+)"
+echo "${fcntl_out}"
+PASS=$((PASS + $(grep -c '\[PASS\]' <<<"${fcntl_out}")))
+FAIL=$((FAIL + $(grep -c '\[FAIL\]' <<<"${fcntl_out}")))
+
+# 2. Distributed lock blocks a cross-mount writer, then hands it off ----------
+# mount 2 holds a file open for writing (holding the DLM lock on its path).
+# An SMB put of the same file goes through mount 1 and must (a) block while
+# mount 2 holds it and (b) succeed once mount 2 releases, leaving the SMB
+# writer's payload on disk. smbclient gets a long client timeout (-t) so we are
+# testing the lock handoff itself, not smbclient's own ~20s default timeout.
+echo "==> 2. distributed lock: cross-mount write coordination"
+dlmfile="dlm_coord.bin"
+newdata="${WORK}/dlm_new.bin"
+head -c 4096 /dev/urandom >"${newdata}"
+
+# Hold the file open for writing on mount 2 via fd 9 -> holds the DLM lock.
+exec 9>"${MOUNT2_SHARE}/${dlmfile}"
+printf 'held-by-mount2' >&9
+
+# Start the SMB write; record its real exit code when it returns. The subshell
+# must NOT inherit fd 9 (9>&-): otherwise the SMB writer keeps the file open and
+# waits on a DLM lock held by its own inherited descriptor, deadlocking the
+# handoff this test is meant to exercise.
+rm -f "${WORK}/dlm_put.rc"
+(
+  smbclient "//${SMB_HOST}/${SMB_SHARE}" -p "${SMB_PORT}" \
+    -U "${SMB_USER}%${SMB_PASS}" -m SMB3 -t 120 \
+    -c "put ${newdata} ${dlmfile}" >/dev/null 2>&1
+  echo "$?" >"${WORK}/dlm_put.rc"
+) 9>&- &
+smb_bg=$!
+
+sleep 4
+if [[ ! -f "${WORK}/dlm_put.rc" ]]; then
+  pass "SMB write blocks while another mount holds the file open"
+else
+  fail "SMB write returned early instead of blocking (rc=$(cat "${WORK}/dlm_put.rc"))"
+fi
+
+# Release mount 2's DLM lock; the blocked SMB write must now complete.
+exec 9>&-
+
+# Wait (bounded) for the SMB put to finish so a stuck handoff fails the test
+# instead of hanging the suite.
+put_rc="timeout"
+for _ in $(seq 1 20); do
+  if [[ -f "${WORK}/dlm_put.rc" ]]; then
+    put_rc="$(cat "${WORK}/dlm_put.rc")"
+    break
+  fi
+  sleep 1
+done
+kill "${smb_bg}" 2>/dev/null
+wait "${smb_bg}" 2>/dev/null
+
+if [[ "${put_rc}" == "0" ]]; then
+  pass "blocked SMB write succeeds after the other mount releases"
+else
+  fail "blocked SMB write succeeds after the other mount releases (rc=${put_rc})"
+fi
+
+# A correct handoff leaves the SMB writer's payload on disk: mount 1 acquired
+# the lock and wrote after mount 2 released.
+got="${WORK}/dlm_got.bin"
+if smb "get ${dlmfile} ${got}" >/dev/null 2>&1 && [[ "$(md5 "${got}")" == "$(md5 "${newdata}")" ]]; then
+  pass "post-release content is the SMB writer's payload (correct handoff)"
+else
+  fail "post-release content is the SMB writer's payload (correct handoff)"
+fi
+
+# 3. Distributed lock integrity: concurrent writers, same file ---------------
+# An SMB writer (mount 1) and a direct writer (mount 2) race on one file. DLM
+# serializes them, so the result must be exactly one of the two payloads.
+echo "==> 3. distributed lock: concurrent writers produce intact data"
+racefile="dlm_race.bin"
+payloadA="${WORK}/dlm_raceA.bin"
+head -c 1048576 /dev/urandom >"${payloadA}"
+payloadB="direct-write-from-mount2-payload"
+(smb "put ${payloadA} ${racefile}" >/dev/null 2>&1) &
+(printf '%s' "${payloadB}" >"${MOUNT2_SHARE}/${racefile}") &
+wait
+racegot="${WORK}/dlm_race_got.bin"
+if smb "get ${racefile} ${racegot}" >/dev/null 2>&1 &&
+  { [[ "$(md5 "${racegot}")" == "$(md5 "${payloadA}")" ]] || [[ "$(cat "${racegot}")" == "${payloadB}" ]]; }; then
+  pass "concurrent same-file writers leave one intact payload"
+else
+  fail "concurrent same-file writers leave one intact payload"
+fi
+
+# 4. Concurrent writers to distinct files ------------------------------------
+echo "==> 4. concurrent writers to distinct files"
+n=6
+declare -a srcs=()
+for i in $(seq 1 "${n}"); do
+  s="${WORK}/cc_${i}.bin"
+  head -c 1048576 /dev/urandom >"${s}"
+  srcs+=("${s}")
+  (smb "put ${s} concurrent_${i}.bin" >/dev/null 2>&1) &
+done
+wait
+all_ok=true
+for i in $(seq 1 "${n}"); do
+  g="${WORK}/cc_got_${i}.bin"
+  if ! smb "get concurrent_${i}.bin ${g}" >/dev/null 2>&1 ||
+    [[ "$(md5 "${srcs[$((i - 1))]}")" != "$(md5 "${g}")" ]]; then
+    all_ok=false
+  fi
+done
+if ${all_ok}; then
+  pass "${n} concurrent distinct-file writes all intact"
+else
+  fail "${n} concurrent distinct-file writes all intact"
+fi
+
+echo
+echo "==> Summary: ${PASS} passed, ${FAIL} failed"
+[[ "${FAIL}" -eq 0 ]]

test/samba/run.sh

@@ -0,0 +1,203 @@
+#!/usr/bin/env bash
+#
+# Run the SMB (Samba) integration test against a SeaweedFS FUSE mount.
+#
+# Pipeline:
+#   1. start a self-contained "weed mini" (master + volume + filer in one)
+#   2. mount the filesystem with "weed mount"
+#   3. export a subdirectory of the mount over SMB with smbd
+#   4. drive the share with smbclient (test/samba/smb_tests.sh)
+#
+# Everything runs as the current user on unprivileged ports, so no sudo is
+# required. State lives under a temp work dir and is removed on exit.
+#
+# Requirements: weed in $PATH, fusermount3, and Samba's smbd / smbclient /
+# smbpasswd (Debian/Ubuntu: apt-get install samba smbclient).
+#
+# Usage:
+#   test/samba/run.sh
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+WEED_BIN="${WEED_BIN:-weed}"
+WORK_DIR="${WORK_DIR:-$(mktemp -d /tmp/seaweedfs-samba.XXXXXX)}"
+MOUNT_DIR="${MOUNT_DIR:-${WORK_DIR}/mnt}"
+MOUNT2_DIR="${MOUNT2_DIR:-${WORK_DIR}/mnt2}"
+DATA_DIR="${DATA_DIR:-${WORK_DIR}/data}"
+LOG_DIR="${LOG_DIR:-${WORK_DIR}/logs}"
+STATE_DIR="${WORK_DIR}/samba"
+SHARE_DIR="${MOUNT_DIR}/share"
+SHARE_DIR2="${MOUNT2_DIR}/share"
+
+FILER_PORT="${FILER_PORT:-28888}"
+FILER_ADDR="127.0.0.1:${FILER_PORT}"
+SMB_PORT="${SMB_PORT:-4450}"
+SMB_SHARE="seaweedfs"
+SMB_USER="${SMB_USER:-$(id -un)}"
+SMB_PASS="${SMB_PASS:-seaweedfs}"
+
+SMBD_BIN="$(command -v smbd || echo /usr/sbin/smbd)"
+SMBPASSWD_BIN="$(command -v smbpasswd || echo /usr/bin/smbpasswd)"
+
+CI_LOG_DIR="/tmp/seaweedfs-samba-logs"
+
+mini_pid=""
+mount_pid=""
+mount2_pid=""
+smbd_pid=""
+
+unmount_dir() {
+  local dir="$1"
+  if mountpoint -q "${dir}" 2>/dev/null; then
+    fusermount3 -u "${dir}" 2>/dev/null ||
+      fusermount -u "${dir}" 2>/dev/null || true
+  fi
+}
+
+cleanup() {
+  set +e
+  if [[ -n "${smbd_pid}" ]] && kill -0 "${smbd_pid}" 2>/dev/null; then
+    kill -TERM "${smbd_pid}" 2>/dev/null || true
+    wait "${smbd_pid}" 2>/dev/null || true
+  fi
+  for p in "${mount_pid}" "${mount2_pid}"; do
+    if [[ -n "${p}" ]] && kill -0 "${p}" 2>/dev/null; then
+      kill -TERM "${p}" 2>/dev/null || true
+      wait "${p}" 2>/dev/null || true
+    fi
+  done
+  unmount_dir "${MOUNT_DIR}"
+  unmount_dir "${MOUNT2_DIR}"
+  if [[ -n "${mini_pid}" ]] && kill -0 "${mini_pid}" 2>/dev/null; then
+    kill -TERM "${mini_pid}" 2>/dev/null || true
+    wait "${mini_pid}" 2>/dev/null || true
+  fi
+  # Copy logs to a fixed path for CI artifact upload.
+  mkdir -p "${CI_LOG_DIR}"
+  cp "${LOG_DIR}"/*.log "${LOG_DIR}"/*.out "${STATE_DIR}/smbd.log" "${CI_LOG_DIR}/" 2>/dev/null || true
+}
+trap cleanup EXIT INT TERM
+
+mkdir -p "${MOUNT_DIR}" "${MOUNT2_DIR}" "${DATA_DIR}" "${LOG_DIR}" \
+  "${STATE_DIR}/private" "${STATE_DIR}/state" "${STATE_DIR}/cache" \
+  "${STATE_DIR}/lock" "${STATE_DIR}/pid" "${STATE_DIR}/ncalrpc"
+
+# --- 1. weed mini -----------------------------------------------------------
+echo "==> Starting weed mini on ${FILER_ADDR}"
+"${WEED_BIN}" mini \
+  -dir="${DATA_DIR}" \
+  -ip=127.0.0.1 \
+  -filer.port="${FILER_PORT}" \
+  -s3=false \
+  -webdav=false \
+  -admin.ui=false \
+  >"${LOG_DIR}/mini.log" 2>&1 &
+mini_pid=$!
+
+for i in $(seq 1 60); do
+  if (echo >"/dev/tcp/127.0.0.1/${FILER_PORT}") 2>/dev/null; then
+    break
+  fi
+  if ! kill -0 "${mini_pid}" 2>/dev/null; then
+    echo "weed mini exited early; log tail:" >&2
+    tail -n 100 "${LOG_DIR}/mini.log" >&2 || true
+    exit 1
+  fi
+  sleep 0.5
+done
+if ! (echo >"/dev/tcp/127.0.0.1/${FILER_PORT}") 2>/dev/null; then
+  echo "weed mini filer did not become reachable within 30s; log tail:" >&2
+  tail -n 100 "${LOG_DIR}/mini.log" >&2 || true
+  exit 1
+fi
+
+# --- 2. weed mount (two mounts, both with -dlm) -----------------------------
+# mount_with_dlm <mountpoint> <logfile> <pid-var-name>
+mount_with_dlm() {
+  local dir="$1" log="$2" pidvar="$3" pid
+  echo "==> Mounting SeaweedFS at ${dir} with -dlm"
+  "${WEED_BIN}" mount \
+    -filer="${FILER_ADDR}" \
+    -dir="${dir}" \
+    -filer.path=/ \
+    -dirAutoCreate \
+    -dlm \
+    >"${log}" 2>&1 &
+  pid=$!
+  printf -v "${pidvar}" '%s' "${pid}"
+  for _ in $(seq 1 60); do
+    if mountpoint -q "${dir}"; then
+      return 0
+    fi
+    if ! kill -0 "${pid}" 2>/dev/null; then
+      echo "weed mount (${dir}) exited early; log tail:" >&2
+      tail -n 100 "${log}" >&2 || true
+      exit 1
+    fi
+    sleep 0.5
+  done
+  echo "FUSE mount ${dir} did not come up within 30s" >&2
+  tail -n 100 "${log}" >&2 || true
+  exit 1
+}
+
+mount_with_dlm "${MOUNT_DIR}" "${LOG_DIR}/mount.log" mount_pid
+mount_with_dlm "${MOUNT2_DIR}" "${LOG_DIR}/mount2.log" mount2_pid
+
+mkdir -p "${SHARE_DIR}"
+
+# --- 3. smbd ----------------------------------------------------------------
+echo "==> Generating smb.conf and starting smbd on port ${SMB_PORT}"
+SMB_CONF="${STATE_DIR}/smb.conf"
+sed -e "s#@SHARE_PATH@#${SHARE_DIR}#g" \
+  -e "s#@STATE_DIR@#${STATE_DIR}#g" \
+  -e "s#@SMB_PORT@#${SMB_PORT}#g" \
+  -e "s#@FORCE_USER@#${SMB_USER}#g" \
+  "${SCRIPT_DIR}/smb.conf.template" >"${SMB_CONF}"
+
+printf '%s\n%s\n' "${SMB_PASS}" "${SMB_PASS}" |
+  "${SMBPASSWD_BIN}" -c "${SMB_CONF}" -a -s "${SMB_USER}"
+
+"${SMBD_BIN}" -F --no-process-group -s "${SMB_CONF}" >"${LOG_DIR}/smbd.out" 2>&1 &
+smbd_pid=$!
+
+for i in $(seq 1 60); do
+  if (echo >"/dev/tcp/127.0.0.1/${SMB_PORT}") 2>/dev/null; then
+    break
+  fi
+  if ! kill -0 "${smbd_pid}" 2>/dev/null; then
+    echo "smbd exited early; log tail:" >&2
+    tail -n 100 "${LOG_DIR}/smbd.out" "${STATE_DIR}/smbd.log" 2>/dev/null >&2 || true
+    exit 1
+  fi
+  sleep 0.5
+done
+if ! (echo >"/dev/tcp/127.0.0.1/${SMB_PORT}") 2>/dev/null; then
+  echo "smbd did not become reachable within 30s; log tail:" >&2
+  tail -n 100 "${LOG_DIR}/smbd.out" "${STATE_DIR}/smbd.log" 2>/dev/null >&2 || true
+  exit 1
+fi
+
+# --- 4. run the test batteries ---------------------------------------------
+rc=0
+
+echo "==> Running SMB functional test battery"
+SMB_HOST=127.0.0.1 \
+  SMB_SHARE="${SMB_SHARE}" \
+  SMB_PORT="${SMB_PORT}" \
+  SMB_USER="${SMB_USER}" \
+  SMB_PASS="${SMB_PASS}" \
+  SHARE_FS_PATH="${SHARE_DIR}" \
+  "${SCRIPT_DIR}/smb_tests.sh" || rc=1
+
+echo "==> Running SMB locking / concurrency test battery"
+SMB_HOST=127.0.0.1 \
+  SMB_SHARE="${SMB_SHARE}" \
+  SMB_PORT="${SMB_PORT}" \
+  SMB_USER="${SMB_USER}" \
+  SMB_PASS="${SMB_PASS}" \
+  MOUNT_SHARE="${SHARE_DIR}" \
+  MOUNT2_SHARE="${SHARE_DIR2}" \
+  "${SCRIPT_DIR}/lock_tests.sh" || rc=1
+
+exit "${rc}"

test/samba/run_inside_container.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# Runs the SMB test batteries inside the samba container against the local smbd,
+# which serves /mnt/seaweedfs/share over a SeaweedFS FUSE mount. A second FUSE
+# mount (/mnt/seaweedfs2) backs the distributed-locking tests.
+# Invoked via: docker compose exec samba /run_inside_container.sh
+set -euo pipefail
+
+export SMB_HOST=127.0.0.1
+export SMB_SHARE=seaweedfs
+export SMB_PORT="${SMB_PORT:-445}"
+export SMB_USER="${SMB_USER:-smbtest}"
+export SMB_PASS="${SMB_PASS:-smbtest}"
+export SHARE_FS_PATH="${SHARE_FS_PATH:-/mnt/seaweedfs/share}"
+export MOUNT_SHARE="${MOUNT_SHARE:-/mnt/seaweedfs/share}"
+export MOUNT2_SHARE="${MOUNT2_SHARE:-/mnt/seaweedfs2/share}"
+
+rc=0
+echo "############ SMB functional tests ############"
+/smb_tests.sh || rc=1
+echo
+echo "############ SMB locking / concurrency tests ############"
+/lock_tests.sh || rc=1
+exit "${rc}"

test/samba/smb.conf.template

@@ -0,0 +1,52 @@
+[global]
+    server role = standalone server
+    workgroup = WORKGROUP
+    server string = SeaweedFS FUSE Samba test
+    security = user
+    server min protocol = SMB2
+    smb ports = @SMB_PORT@
+    bind interfaces only = yes
+    interfaces = lo 127.0.0.1
+
+    # Self-contained state so smbd can run rootless and leaves nothing behind
+    # outside the test work directory.
+    private dir = @STATE_DIR@/private
+    state directory = @STATE_DIR@/state
+    cache directory = @STATE_DIR@/cache
+    lock directory = @STATE_DIR@/lock
+    pid directory = @STATE_DIR@/pid
+    ncalrpc dir = @STATE_DIR@/ncalrpc
+    log file = @STATE_DIR@/smbd.log
+    log level = 1
+    usershare max shares = 0
+
+    # No printing subsystem in a file-server test.
+    load printers = no
+    printing = bsd
+    printcap name = /dev/null
+    disable spoolss = yes
+
+    # The SeaweedFS FUSE mount does not implement the full xattr / DOS-attribute
+    # surface Samba uses by default. Disabling these avoids spurious
+    # NT_STATUS_NOT_SUPPORTED / EOPNOTSUPP errors unrelated to data integrity.
+    ea support = no
+    store dos attributes = no
+    map archive = no
+    map hidden = no
+    map system = no
+    map readonly = no
+
+    # A network-filesystem backend should not advertise local oplocks/leases.
+    oplocks = no
+    level2 oplocks = no
+    kernel oplocks = no
+    posix locking = no
+
+[seaweedfs]
+    path = @SHARE_PATH@
+    comment = SeaweedFS share backed by a FUSE mount
+    browseable = yes
+    read only = no
+    create mask = 0644
+    directory mask = 0755
+    force user = @FORCE_USER@

test/samba/smb_tests.sh

@@ -0,0 +1,172 @@
+#!/usr/bin/env bash
+#
+# SMB protocol test battery against a Samba share backed by a SeaweedFS FUSE
+# mount. Driven both by the local runner (test/samba/run.sh) and by the Docker
+# harness (run_inside_container.sh).
+#
+# Required env:
+#   SMB_USER   samba username
+#   SMB_PASS   samba password
+# Optional env:
+#   SMB_HOST   samba host                (default 127.0.0.1)
+#   SMB_SHARE  share name                (default seaweedfs)
+#   SMB_PORT   smbd port                 (default 445)
+#   SHARE_FS_PATH  directory on the FUSE mount that backs the share. When set,
+#                  the suite also checks cross-protocol consistency: data written
+#                  over SMB is visible on the FUSE mount, and vice versa.
+set -uo pipefail
+
+SMB_HOST="${SMB_HOST:-127.0.0.1}"
+SMB_SHARE="${SMB_SHARE:-seaweedfs}"
+SMB_PORT="${SMB_PORT:-445}"
+SMB_USER="${SMB_USER:?SMB_USER is required}"
+SMB_PASS="${SMB_PASS:?SMB_PASS is required}"
+SHARE_FS_PATH="${SHARE_FS_PATH:-}"
+
+WORK="$(mktemp -d /tmp/samba-smbtest.XXXXXX)"
+trap 'rm -rf "${WORK}"' EXIT
+
+PASS=0
+FAIL=0
+pass() { printf '  [PASS] %s\n' "$1"; PASS=$((PASS + 1)); }
+fail() { printf '  [FAIL] %s\n' "$1"; FAIL=$((FAIL + 1)); }
+
+# Run one or more smbclient commands (separated by ';') against the share.
+smb() {
+  smbclient "//${SMB_HOST}/${SMB_SHARE}" -p "${SMB_PORT}" \
+    -U "${SMB_USER}%${SMB_PASS}" -m SMB3 -c "$1"
+}
+
+md5() { md5sum "$1" | awk '{print $1}'; }
+
+echo "==> Target //${SMB_HOST}/${SMB_SHARE} (port ${SMB_PORT}) as ${SMB_USER}"
+[[ -n "${SHARE_FS_PATH}" ]] && echo "==> Cross-protocol checks against ${SHARE_FS_PATH}"
+
+# 1. Connectivity ------------------------------------------------------------
+echo "==> 1. connectivity"
+if smb "ls" >/dev/null 2>&1; then
+  pass "connect and list share root"
+else
+  fail "connect and list share root"
+fi
+
+# 2. Upload / download round-trip -------------------------------------------
+echo "==> 2. upload / download round-trip"
+src="${WORK}/src.bin"
+head -c 1048576 /dev/urandom >"${src}" # 1 MiB
+if smb "put ${src} roundtrip.bin" >/dev/null 2>&1; then
+  pass "put 1 MiB file"
+else
+  fail "put 1 MiB file"
+fi
+got="${WORK}/got.bin"
+if smb "get roundtrip.bin ${got}" >/dev/null 2>&1 && [[ "$(md5 "${src}")" == "$(md5 "${got}")" ]]; then
+  pass "get returns identical content"
+else
+  fail "get returns identical content"
+fi
+if [[ -n "${SHARE_FS_PATH}" ]]; then
+  if [[ -f "${SHARE_FS_PATH}/roundtrip.bin" ]] && [[ "$(md5 "${SHARE_FS_PATH}/roundtrip.bin")" == "$(md5 "${src}")" ]]; then
+    pass "SMB-written file visible on FUSE mount with identical content"
+  else
+    fail "SMB-written file visible on FUSE mount with identical content"
+  fi
+fi
+
+# 3. Directory operations ----------------------------------------------------
+echo "==> 3. directory operations"
+if smb "mkdir docs; cd docs; put ${src} nested.bin; ls" >/dev/null 2>&1; then
+  pass "mkdir + put into subdirectory"
+else
+  fail "mkdir + put into subdirectory"
+fi
+if [[ -z "${SHARE_FS_PATH}" || -f "${SHARE_FS_PATH}/docs/nested.bin" ]]; then
+  pass "nested file present"
+else
+  fail "nested file present"
+fi
+
+# 4. Rename ------------------------------------------------------------------
+echo "==> 4. rename"
+if smb "rename roundtrip.bin renamed.bin" >/dev/null 2>&1; then
+  pass "rename file"
+else
+  fail "rename file"
+fi
+renback="${WORK}/renamed.bin"
+if smb "get renamed.bin ${renback}" >/dev/null 2>&1 && [[ "$(md5 "${renback}")" == "$(md5 "${src}")" ]]; then
+  pass "renamed file readable with original content"
+else
+  fail "renamed file readable with original content"
+fi
+if [[ -n "${SHARE_FS_PATH}" ]]; then
+  if [[ -f "${SHARE_FS_PATH}/renamed.bin" && ! -e "${SHARE_FS_PATH}/roundtrip.bin" ]]; then
+    pass "rename reflected on FUSE mount"
+  else
+    fail "rename reflected on FUSE mount"
+  fi
+fi
+
+# 5. Large file (exercises SeaweedFS chunking) -------------------------------
+echo "==> 5. large file (SeaweedFS chunking)"
+big="${WORK}/big.bin"
+head -c 67108864 /dev/urandom >"${big}" # 64 MiB
+bigback="${WORK}/big.back"
+if smb "put ${big} big.bin" >/dev/null 2>&1 &&
+  smb "get big.bin ${bigback}" >/dev/null 2>&1 &&
+  [[ "$(md5 "${big}")" == "$(md5 "${bigback}")" ]]; then
+  pass "64 MiB put/get round-trip"
+else
+  fail "64 MiB put/get round-trip"
+fi
+
+# 6. Recursive upload --------------------------------------------------------
+echo "==> 6. recursive upload"
+tree="${WORK}/tree"
+mkdir -p "${tree}/a/b"
+echo one >"${tree}/f1.txt"
+echo two >"${tree}/a/f2.txt"
+echo three >"${tree}/a/b/f3.txt"
+if (cd "${WORK}" && smb "recurse ON; prompt OFF; mput tree" >/dev/null 2>&1) &&
+  { [[ -z "${SHARE_FS_PATH}" ]] || [[ -f "${SHARE_FS_PATH}/tree/a/b/f3.txt" ]]; }; then
+  pass "recursive mput"
+else
+  fail "recursive mput"
+fi
+
+# 7. Cross-protocol read (FUSE writes, SMB reads) ----------------------------
+if [[ -n "${SHARE_FS_PATH}" ]]; then
+  echo "==> 7. cross-protocol read (FUSE write -> SMB read)"
+  echo "written-via-fuse" >"${SHARE_FS_PATH}/from_fuse.txt"
+  cpb="${WORK}/from_fuse.back"
+  if smb "get from_fuse.txt ${cpb}" >/dev/null 2>&1 && grep -q written-via-fuse "${cpb}"; then
+    pass "FUSE-written file readable over SMB"
+  else
+    fail "FUSE-written file readable over SMB"
+  fi
+fi
+
+# 8. Delete ------------------------------------------------------------------
+echo "==> 8. delete"
+smb "del renamed.bin" >/dev/null 2>&1
+smb "del big.bin" >/dev/null 2>&1
+smb "deltree docs" >/dev/null 2>&1
+smb "deltree tree" >/dev/null 2>&1
+if [[ -n "${SHARE_FS_PATH}" ]]; then
+  if [[ ! -e "${SHARE_FS_PATH}/renamed.bin" && ! -e "${SHARE_FS_PATH}/big.bin" &&
+    ! -e "${SHARE_FS_PATH}/docs" && ! -e "${SHARE_FS_PATH}/tree" ]]; then
+    pass "delete files and directory trees"
+  else
+    fail "delete files and directory trees"
+  fi
+else
+  if ! smb "get renamed.bin /dev/null" >/dev/null 2>&1; then
+    pass "deleted file no longer retrievable"
+  else
+    fail "deleted file no longer retrievable"
+  fi
+fi
+
+echo
+echo "==> Summary: ${PASS} passed, ${FAIL} failed"
+[[ "${FAIL}" -eq 0 ]]

test/testutil/ports.go

@@ -11,6 +11,29 @@ import (
 // GrpcPortOffset is the offset weed mini uses to derive gRPC ports from HTTP ports.
 const GrpcPortOffset = 10000
 
+// miniDefaultPorts are the weed mini flag defaults (see weed/command/mini.go).
+// A test only overrides services it uses; unspecified services still bind
+// these defaults, so allocation must avoid handing them out (or any value
+// whose gRPC offset would collide with them).
+var miniDefaultPorts = []int{
+	9333,  // master.port
+	8888,  // filer.port
+	9340,  // volume.port
+	8333,  // s3.port
+	8181,  // s3.port.iceberg
+	7333,  // webdav.port
+	23646, // admin.port
+}
+
+func reservedMiniPorts() map[int]bool {
+	r := make(map[int]bool, len(miniDefaultPorts)*2)
+	for _, p := range miniDefaultPorts {
+		r[p] = true
+		r[p+GrpcPortOffset] = true
+	}
+	return r
+}
+
 // AllocatePorts allocates count unique free ports atomically.
 // All listeners are held open until every port is obtained, preventing
 // the OS from recycling a port between successive allocations.
@@ -51,12 +74,19 @@ func MustAllocatePorts(t *testing.T, count int) []int {
 // from recycling ports between allocations. Use this when ports will be
 // passed to weed mini without explicit gRPC port flags, so mini will
 // derive gRPC ports as HTTP + 10000.
+//
+// Listeners are bound on all interfaces (":port") rather than 127.0.0.1
+// to match weed mini's availability check (isPortAvailable). A port can
+// be free on loopback but held by another process on a different
+// interface; reserving only on loopback lets mini's check fail and
+// trigger gRPC port shifting, which then causes weed shell to dial the
+// wrong port and hang.
 func AllocateMiniPorts(count int) ([]int, error) {
 	const (
 		minPort = 10000
 		maxPort = 55000
 	)
-	reserved := make(map[int]bool)
+	reserved := reservedMiniPorts()
 	ports := make([]int, 0, count)
 	var listeners []net.Listener
 	defer func() {
@@ -75,12 +105,12 @@ func AllocateMiniPorts(count int) ([]int, error) {
 				continue
 			}
 
-			l1, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+			l1, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
 			if err != nil {
 				continue
 			}
 
-			l2, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
+			l2, err := net.Listen("tcp", fmt.Sprintf(":%d", grpcPort))
 			if err != nil {
 				l1.Close()
 				continue
@@ -128,7 +158,7 @@ func AllocatePortSet(miniCount, regularCount int) (mini []int, regular []int, er
 		minPort = 10000
 		maxPort = 55000
 	)
-	reserved := make(map[int]bool)
+	reserved := reservedMiniPorts()
 	mini = make([]int, 0, miniCount)
 	var listeners []net.Listener
 	defer func() {
@@ -145,11 +175,11 @@ func AllocatePortSet(miniCount, regularCount int) (mini []int, regular []int, er
 			if reserved[port] || reserved[grpcPort] {
 				continue
 			}
-			l1, lErr := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+			l1, lErr := net.Listen("tcp", fmt.Sprintf(":%d", port))
 			if lErr != nil {
 				continue
 			}
-			l2, lErr := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
+			l2, lErr := net.Listen("tcp", fmt.Sprintf(":%d", grpcPort))
 			if lErr != nil {
 				l1.Close()
 				continue
@@ -168,7 +198,7 @@ func AllocatePortSet(miniCount, regularCount int) (mini []int, regular []int, er
 
 	regular = make([]int, 0, regularCount)
 	for i := 0; i < regularCount; i++ {
-		l, lErr := net.Listen("tcp", "127.0.0.1:0")
+		l, lErr := net.Listen("tcp", ":0")
 		if lErr != nil {
 			return nil, nil, lErr
 		}

test/testutil/ports_test.go

@@ -2,6 +2,29 @@ package testutil
 
 import "testing"
 
+// AllocateMiniPorts must never hand out a port that weed mini will reserve
+// for one of its default services (or that default's gRPC offset). A real
+// failure: Filer was given 33646 (Admin default 23646 + GrpcPortOffset),
+// which mini then refused as "reserved for gRPC calculation".
+func TestAllocateMiniPortsAvoidsMiniDefaults(t *testing.T) {
+	reserved := reservedMiniPorts()
+	for iter := 0; iter < 200; iter++ {
+		ports, err := AllocateMiniPorts(4)
+		if err != nil {
+			t.Fatalf("iter %d: AllocateMiniPorts: %v", iter, err)
+		}
+		for _, p := range ports {
+			if reserved[p] {
+				t.Fatalf("iter %d: allocated port %d is a mini default (or gRPC offset)", iter, p)
+			}
+			if reserved[p+GrpcPortOffset] {
+				t.Fatalf("iter %d: allocated port %d has gRPC offset %d colliding with a mini default",
+					iter, p, p+GrpcPortOffset)
+			}
+		}
+	}
+}
+
 func TestAllocatePortSetNoGrpcCollision(t *testing.T) {
 	// Run a few iterations to catch the OS-recycles-just-closed-port race
 	// that previously hit regular ports when the mini gRPC offset was freed

test/volume_server/grpc/ec_multi_disk_lifecycle_test.go

@@ -136,8 +136,6 @@ func TestEcLifecycleAcrossMultipleDisks(t *testing.T) {
 	conn2, grpcClient2 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
 	defer conn2.Close()
 
-	// VolumeEcShardsInfo only sees one disk's EcVolume; filesystem layout is
-	// the ground truth for the whole-store shard count.
 	postReconcileLayout := scanShardLayout(t, dataDirs, collection, volumeID)
 	if got, want := totalShardsInLayout(postReconcileLayout), erasure_coding.TotalShardsCount; got != want {
 		t.Fatalf("post-reconcile: total shards on disk mismatch: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
@@ -148,11 +146,28 @@ func TestEcLifecycleAcrossMultipleDisks(t *testing.T) {
 	if got, want := len(postReconcileLayout[1]), splitAt; got != want {
 		t.Fatalf("post-reconcile: disk 1 shard count drift: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
 	}
-	if _, err := grpcClient2.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+	// VolumeEcShardsInfo must walk every DiskLocation and report the full
+	// shard set — the verification step in ec_task.go gates source-volume
+	// deletion on this RPC returning a complete shard inventory.
+	infoResp, err := grpcClient2.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
 		VolumeId: volumeID,
-	}); err != nil {
+	})
+	if err != nil {
 		t.Fatalf("VolumeEcShardsInfo after redistribute restart: %v", err)
 	}
+	if got, want := len(infoResp.GetEcShardInfos()), erasure_coding.TotalShardsCount; got != want {
+		t.Fatalf("VolumeEcShardsInfo after redistribute restart: got %d shards, want %d (per-disk layout=%v)",
+			got, want, postReconcileLayout)
+	}
+	gotShardIds := make(map[uint32]struct{}, len(infoResp.GetEcShardInfos()))
+	for _, info := range infoResp.GetEcShardInfos() {
+		gotShardIds[info.GetShardId()] = struct{}{}
+	}
+	for shardId := uint32(0); shardId < uint32(erasure_coding.TotalShardsCount); shardId++ {
+		if _, ok := gotShardIds[shardId]; !ok {
+			t.Fatalf("VolumeEcShardsInfo missing shard %d (per-disk layout=%v)", shardId, postReconcileLayout)
+		}
+	}
 	for _, n := range needles {
 		verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-cross-disk-reconcile")
 	}
@@ -205,8 +220,21 @@ func TestEcLifecycleAcrossMultipleDisks(t *testing.T) {
 		t.Fatalf("VolumeEcShardsRebuild expected to rebuild shard %d, got %v",
 			repairTargetShard, rebuildResp.GetRebuiltShardIds())
 	}
-	if _, statErr := os.Stat(shardPath); statErr != nil {
-		t.Fatalf("rebuild did not restore shard %d on disk 0 (%s): %v", repairTargetShard, shardPath, statErr)
+	// The rebuilder picks whichever disk hosts the most shards plus a
+	// matching .ecx. After the boot-time mirror runs, every shard-
+	// bearing disk owns its own sidecars, so rebuild can legitimately
+	// restore the shard on either disk — accept both.
+	rebuiltOn := -1
+	for i, dir := range dataDirs {
+		candidate := filepath.Join(dir, shardFileName(collection, volumeID, repairTargetShard))
+		if _, statErr := os.Stat(candidate); statErr == nil {
+			rebuiltOn = i
+			shardPath = candidate
+			break
+		}
+	}
+	if rebuiltOn < 0 {
+		t.Fatalf("rebuild did not restore shard %d on any disk (checked %v)", repairTargetShard, dataDirs)
 	}
 
 	if _, err := grpcClient3.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{

test/volume_server/grpc/ec_verify_multi_disk_test.go

@@ -0,0 +1,175 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"net/http"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// TestVolumeEcShardsInfoReturnsAllShardsAcrossDisks drives the full path
+// behind the ec.encode source-deletion gate. A multi-disk volume server
+// ends up with EC shards split across disks (each registers its own
+// EcVolume entry in DiskLocation.ecVolumes), and the volume server's
+// VolumeEcShardsInfo RPC must walk every DiskLocation rather than
+// reporting whichever disk Store.FindEcVolume picks first.
+//
+// Pre-fix, verifyEcShardsBeforeDelete refused to delete source volumes —
+// the shard-bitmap union across destinations fell short of dataShards +
+// parityShards because each destination only reported shards on one of
+// its disks. With the handler fix, the same VerifyShardsAcrossServers
+// call returns a complete bitmap and the gate opens.
+func TestVolumeEcShardsInfoReturnsAllShardsAcrossDisks(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	const (
+		dataDirCount = 2
+		volumeID     = uint32(9558)
+		collection   = "ec-multi-disk-verify"
+	)
+
+	clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
+	dataDirs := clusterHarness.VolumeDataDirs()
+	if len(dataDirs) != dataDirCount {
+		t.Fatalf("expected %d data dirs, got %d: %v", dataDirCount, len(dataDirs), dataDirs)
+	}
+
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	framework.AllocateVolume(t, grpcClient, volumeID, collection)
+
+	httpClient := framework.NewHTTPClient()
+	needles := []struct {
+		fid     string
+		payload []byte
+	}{
+		{framework.NewFileID(volumeID, 9559, 0xC0FFEE01), bytesOfLen(64, 0xB1)},
+		{framework.NewFileID(volumeID, 9560, 0xC0FFEE02), bytesOfLen(8192, 0xB2)},
+		{framework.NewFileID(volumeID, 9561, 0xC0FFEE03), bytesOfLen(131072, 0xB3)},
+	}
+	for _, n := range needles {
+		resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload)
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s expected 201, got %d", n.fid, resp.StatusCode)
+		}
+	}
+
+	if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: collection,
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsGenerate: %v", err)
+	}
+
+	// Generate places every shard plus the .ecx/.ecj/.vif on the .dat's
+	// disk (disk 0). Mount all 14 there first so the next step's restart
+	// has a steady starting state.
+	allShards := make([]uint32, erasure_coding.TotalShardsCount)
+	for i := range allShards {
+		allShards[i] = uint32(i)
+	}
+	if _, err := grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: collection,
+		ShardIds:   allShards,
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsMount all shards: %v", err)
+	}
+
+	// Drop the .dat so the EC shards are the only data path — mirrors the
+	// real ec.encode flow before verifyEcShardsBeforeDelete fires.
+	if _, err := grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
+		VolumeId: volumeID,
+	}); err != nil {
+		t.Fatalf("VolumeDelete (drop .dat): %v", err)
+	}
+
+	// Move half the shards onto disk 1, leaving .ecx on disk 0. After
+	// restart, the cross-disk reconcile path attaches each disk's shards
+	// against its own EcVolume entry — the exact in-memory shape the bug
+	// reporter saw on a multi-disk destination.
+	clusterHarness.StopVolumeServer()
+	const splitAt = 7
+	for shard := 0; shard < splitAt; shard++ {
+		movedFile(t, dataDirs[0], dataDirs[1], collection, volumeID, erasure_coding.ToExt(shard))
+	}
+	if fileExistsIn(dataDirs[1], collection, volumeID, ".ecx") {
+		t.Fatalf("setup: .ecx must stay on disk 0 to exercise the multi-disk path")
+	}
+
+	clusterHarness.RestartVolumeServer()
+	conn2, grpcClient2 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn2.Close()
+
+	postReconcileLayout := scanShardLayout(t, dataDirs, collection, volumeID)
+	if got, want := totalShardsInLayout(postReconcileLayout), erasure_coding.TotalShardsCount; got != want {
+		t.Fatalf("post-reconcile: total shards on disk mismatch: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
+	}
+	if len(postReconcileLayout[0]) == 0 || len(postReconcileLayout[1]) == 0 {
+		t.Fatalf("post-reconcile: expected shards on BOTH disks, got per-disk layout %v", postReconcileLayout)
+	}
+
+	// Direct RPC assertion: VolumeEcShardsInfo must report every shard
+	// the server holds, not just the ones registered against the first
+	// matching DiskLocation.
+	infoResp, err := grpcClient2.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsInfo: %v", err)
+	}
+	gotShardIds := make([]int, 0, len(infoResp.GetEcShardInfos()))
+	for _, info := range infoResp.GetEcShardInfos() {
+		if info.GetVolumeId() != volumeID {
+			t.Errorf("EcShardInfo VolumeId=%d, want %d", info.GetVolumeId(), volumeID)
+		}
+		gotShardIds = append(gotShardIds, int(info.GetShardId()))
+	}
+	sort.Ints(gotShardIds)
+	wantShardIds := make([]int, erasure_coding.TotalShardsCount)
+	for i := range wantShardIds {
+		wantShardIds[i] = i
+	}
+	if len(gotShardIds) != len(wantShardIds) {
+		t.Fatalf("VolumeEcShardsInfo returned %d shards (ids=%v), want %d (ids=%v) — per-disk layout=%v",
+			len(gotShardIds), gotShardIds, len(wantShardIds), wantShardIds, postReconcileLayout)
+	}
+	for i, sid := range wantShardIds {
+		if gotShardIds[i] != sid {
+			t.Fatalf("VolumeEcShardsInfo shard ids=%v, want %v (per-disk layout=%v)",
+				gotShardIds, wantShardIds, postReconcileLayout)
+		}
+	}
+
+	// End-to-end assertion via the same helper the worker uses to gate
+	// source-volume deletion (weed/worker/tasks/erasure_coding/ec_task.go
+	// verifyEcShardsBeforeDelete). The union across destinations is what
+	// RequireFullShardSet measures; with one destination that holds every
+	// shard, the union must cover dataShards + parityShards.
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	servers := []string{clusterHarness.VolumeServerAddress()}
+	union, perServer := erasure_coding.VerifyShardsAcrossServers(ctx, volumeID, servers, dialOption)
+	if err := erasure_coding.RequireFullShardSet(volumeID, union, erasure_coding.TotalShardsCount); err != nil {
+		t.Fatalf("verifyEcShardsBeforeDelete-equivalent gate failed: %v\nper-server inventory: %s\nper-disk layout: %v",
+			err, erasure_coding.SummarizeShardInventory(perServer), postReconcileLayout)
+	}
+	if got, want := union.Count(), erasure_coding.TotalShardsCount; got != want {
+		t.Fatalf("VerifyShardsAcrossServers union covered %d/%d shards (per-server=%s, layout=%v)",
+			got, want, erasure_coding.SummarizeShardInventory(perServer), postReconcileLayout)
+	}
+}

test/volume_server/http/production_features_test.go

@@ -75,7 +75,7 @@ func TestStatsEndpoints(t *testing.T) {
 	}
 }
 
-func TestStatusPrettyJsonAndJsonp(t *testing.T) {
+func TestStatusPrettyJsonAndCallbackIgnored(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test in short mode")
 	}
@@ -93,29 +93,29 @@ func TestStatusPrettyJsonAndJsonp(t *testing.T) {
 	if len(lines) < 3 {
 		t.Fatalf("/status?pretty=y expected multi-line indented JSON, got %d lines: %s", len(lines), string(prettyBody))
 	}
-	// Verify the body is valid JSON
 	var prettyPayload map[string]interface{}
 	if err := json.Unmarshal(prettyBody, &prettyPayload); err != nil {
 		t.Fatalf("/status?pretty=y is not valid JSON: %v", err)
 	}
 
-	// ?callback=myFunc — expect JSONP wrapping
-	jsonpResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/status?callback=myFunc"))
-	jsonpBody := framework.ReadAllAndClose(t, jsonpResp)
-	if jsonpResp.StatusCode != http.StatusOK {
-		t.Fatalf("/status?callback=myFunc expected 200, got %d", jsonpResp.StatusCode)
+	// ?callback=myFunc — must be ignored; response is plain JSON with nosniff.
+	cbResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/status?callback=myFunc"))
+	cbBody := framework.ReadAllAndClose(t, cbResp)
+	if cbResp.StatusCode != http.StatusOK {
+		t.Fatalf("/status?callback=myFunc expected 200, got %d", cbResp.StatusCode)
 	}
-	bodyStr := string(jsonpBody)
-	if !strings.HasPrefix(bodyStr, "myFunc(") {
-		t.Fatalf("/status?callback=myFunc expected body to start with 'myFunc(', got prefix: %q", bodyStr[:min(len(bodyStr), 30)])
+	if ct := cbResp.Header.Get("Content-Type"); !strings.Contains(ct, "application/json") {
+		t.Fatalf("/status?callback=myFunc expected Content-Type application/json, got %q", ct)
 	}
-	trimmed := strings.TrimRight(bodyStr, "\n; ")
-	if !strings.HasSuffix(trimmed, ")") {
-		t.Fatalf("/status?callback=myFunc expected body to end with ')', got suffix: %q", trimmed[max(0, len(trimmed)-10):])
+	if nosniff := cbResp.Header.Get("X-Content-Type-Options"); nosniff != "nosniff" {
+		t.Fatalf("/status?callback=myFunc expected X-Content-Type-Options nosniff, got %q", nosniff)
 	}
-	// Content-Type should be application/javascript for JSONP
-	if ct := jsonpResp.Header.Get("Content-Type"); !strings.Contains(ct, "javascript") {
-		t.Fatalf("/status?callback=myFunc expected Content-Type containing 'javascript', got %q", ct)
+	if strings.Contains(string(cbBody), "myFunc(") {
+		t.Fatalf("/status?callback=myFunc must not wrap response in callback; body: %q", string(cbBody))
+	}
+	var cbPayload map[string]interface{}
+	if err := json.Unmarshal(cbBody, &cbPayload); err != nil {
+		t.Fatalf("/status?callback=myFunc is not valid JSON: %v", err)
 	}
 }
 

test/volume_server/http/replication_lifecycle_test.go

@@ -2,6 +2,8 @@ package volume_server_http_test
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"net/http"
 	"testing"
 	"time"
@@ -44,6 +46,17 @@ func TestReplicatedUploadSucceedsImmediatelyAfterAllocate(t *testing.T) {
 	fid := framework.NewFileID(volumeID, 881001, 0x0B0C0D0E)
 	payload := []byte("replicated-upload-after-allocate")
 
+	// The master only learns about replica locations through volume-server
+	// heartbeats, which lag behind the direct AllocateVolume gRPC calls above.
+	// In production a client obtains its fid from the master assign flow, which
+	// guarantees the master already knows every replica; this test crafts the
+	// fid by hand, so the replicated write would otherwise look up the master
+	// before the second replica is registered and fail with a 500. Wait until
+	// the master reports both replicas before uploading.
+	if !waitForMasterReplicaCount(t, client, clusterHarness.MasterURL(), volumeID, 2, 10*time.Second) {
+		t.Fatalf("master did not report 2 replica locations for volume %d within deadline", volumeID)
+	}
+
 	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload)
 	_ = framework.ReadAllAndClose(t, uploadResp)
 	if uploadResp.StatusCode != http.StatusCreated {
@@ -61,3 +74,29 @@ func TestReplicatedUploadSucceedsImmediatelyAfterAllocate(t *testing.T) {
 		t.Fatalf("replica body mismatch: got %q want %q", string(replicaBody), string(payload))
 	}
 }
+
+// waitForMasterReplicaCount polls the master volume lookup until it reports at
+// least want locations for volumeID, or the timeout elapses.
+func waitForMasterReplicaCount(t testing.TB, client *http.Client, masterURL string, volumeID uint32, want int, timeout time.Duration) bool {
+	t.Helper()
+
+	lookupURL := fmt.Sprintf("%s/dir/lookup?volumeId=%d", masterURL, volumeID)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, lookupURL))
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode == http.StatusOK {
+			var result struct {
+				Locations []struct {
+					Url string `json:"url"`
+				} `json:"locations"`
+			}
+			if err := json.Unmarshal(body, &result); err == nil && len(result.Locations) >= want {
+				return true
+			}
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+
+	return false
+}

weed/admin/dash/admin_server.go

@@ -23,6 +23,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
+	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -194,12 +195,10 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 				glog.V(0).Infof("Credential store %s does not support filer address function", store.GetName())
 			}
 
-			// The filer's IAM gRPC service rejects every RPC without an
-			// admin-signed Bearer token (PR #9442). Mirror the filer's
-			// jwt.filer_signing.key here so the admin UI's Users/Groups
-			// pages can talk to it; without this they fail with either
-			// Unimplemented (filer refuses to register the service) or
-			// Unauthenticated (missing authorization metadata).
+			// Mirror the filer's jwt.filer_signing.key so the admin UI's
+			// Users/Groups pages can present a valid Bearer token when the
+			// filer enforces IAM gRPC auth. When the key is empty, both
+			// sides run unauthenticated and no token is sent.
 			if signer, ok := store.(interface {
 				SetAdminSigning(security.SigningKey, int)
 			}); ok {
@@ -207,9 +206,7 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 				key := security.SigningKey(viper.GetString("jwt.filer_signing.key"))
 				expires := viper.GetInt("jwt.filer_signing.expires_after_seconds")
 				signer.SetAdminSigning(key, expires)
-				if len(key) == 0 {
-					glog.Warningf("jwt.filer_signing.key is empty in security.toml; the admin UI Users/Groups pages will fail until this is set on both the filer and the admin server")
-				} else {
+				if len(key) > 0 {
 					glog.V(0).Infof("Credential store configured with admin Bearer token signing")
 				}
 			}
@@ -284,6 +281,8 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 		go server.monitorVacuumWorker(bgCtx)
 	}
 
+	go server.publishMaintenanceMetrics(bgCtx)
+
 	return server
 }
 
@@ -368,6 +367,55 @@ func (s *AdminServer) monitorVacuumWorker(ctx context.Context) {
 	}
 }
 
+// publishMaintenanceMetrics periodically snapshots the maintenance queue and
+// worker fleet into Prometheus gauges. Counters and durations are recorded at
+// their event sites; these gauges reflect current state at scrape resolution.
+func (s *AdminServer) publishMaintenanceMetrics(ctx context.Context) {
+	const interval = 15 * time.Second
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			s.collectMaintenanceMetrics()
+		}
+	}
+}
+
+func (s *AdminServer) collectMaintenanceMetrics() {
+	if s.maintenanceManager == nil {
+		return
+	}
+
+	stats := s.maintenanceManager.GetStats()
+
+	stats_collect.AdminMaintenanceTasksByStatus.Reset()
+	for status, count := range stats.TasksByStatus {
+		stats_collect.AdminMaintenanceTasksByStatus.WithLabelValues(string(status)).Set(float64(count))
+	}
+
+	stats_collect.AdminMaintenanceTasksByType.Reset()
+	for taskType, count := range stats.TasksByType {
+		stats_collect.AdminMaintenanceTasksByType.WithLabelValues(string(taskType)).Set(float64(count))
+	}
+
+	// NextScanTime is only meaningful while the scanner runs; GetStats computes
+	// it unconditionally, so clear the gauge when idle to avoid a stale value.
+	if s.maintenanceManager.IsRunning() && !stats.NextScanTime.IsZero() {
+		stats_collect.AdminMaintenanceNextScanTimestampSeconds.Set(float64(stats.NextScanTime.Unix()))
+	} else {
+		stats_collect.AdminMaintenanceNextScanTimestampSeconds.Set(0)
+	}
+
+	workers, usedSlots, maxSlots := s.maintenanceManager.GetWorkerSlotTotals()
+	stats_collect.AdminWorkersConnected.Set(float64(workers))
+	stats_collect.AdminWorkerSlots.WithLabelValues("used").Set(float64(usedSlots))
+	stats_collect.AdminWorkerSlots.WithLabelValues("max").Set(float64(maxSlots))
+}
+
 // loadTaskConfigurationsFromPersistence loads saved task configurations from protobuf files
 func (s *AdminServer) loadTaskConfigurationsFromPersistence() {
 	if s.configPersistence == nil || !s.configPersistence.IsConfigured() {

weed/admin/dash/client_management.go

@@ -12,6 +12,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"google.golang.org/grpc"
 )
 
 // WithMasterClient executes a function with a master client connection
@@ -38,6 +40,21 @@ func (s *AdminServer) WithVolumeServerClient(address pb.ServerAddress, f func(cl
 	})
 }
 
+// GetMasterClient returns the admin server's wdclient.MasterClient. It is used
+// by file browser download paths that stream chunks straight from the volume
+// servers via filer.PrepareStreamContent so they keep working when the filer
+// has -disableHttp=true.
+func (s *AdminServer) GetMasterClient() *wdclient.MasterClient {
+	return s.masterClient
+}
+
+// GetGrpcDialOption returns the dial option used for all admin-originated
+// gRPC connections (TLS or insecure). File browser uploads need this when
+// they perform the assign + volume HTTP POST + create-entry flow.
+func (s *AdminServer) GetGrpcDialOption() grpc.DialOption {
+	return s.grpcDialOption
+}
+
 // GetFilerAddress returns a filer address, discovering from masters if needed
 func (s *AdminServer) GetFilerAddress() string {
 	// Discover filers from masters

weed/admin/dash/plugin_api.go

@@ -734,8 +734,8 @@ func (s *AdminServer) parseOrBuildClusterContext(raw json.RawMessage) (*plugin_p
 	if len(contextMessage.MasterGrpcAddresses) == 0 {
 		contextMessage.MasterGrpcAddresses = append(contextMessage.MasterGrpcAddresses, fallback.MasterGrpcAddresses...)
 	}
-	if len(contextMessage.FilerGrpcAddresses) == 0 {
-		contextMessage.FilerGrpcAddresses = append(contextMessage.FilerGrpcAddresses, fallback.FilerGrpcAddresses...)
+	if len(contextMessage.FilerAddresses) == 0 {
+		contextMessage.FilerAddresses = append(contextMessage.FilerAddresses, fallback.FilerAddresses...)
 	}
 	if len(contextMessage.VolumeGrpcAddresses) == 0 {
 		contextMessage.VolumeGrpcAddresses = append(contextMessage.VolumeGrpcAddresses, fallback.VolumeGrpcAddresses...)
@@ -754,7 +754,7 @@ func (s *AdminServer) parseOrBuildClusterContext(raw json.RawMessage) (*plugin_p
 func (s *AdminServer) buildDefaultPluginClusterContext() *plugin_pb.ClusterContext {
 	clusterContext := &plugin_pb.ClusterContext{
 		MasterGrpcAddresses: make([]string, 0),
-		FilerGrpcAddresses:  make([]string, 0),
+		FilerAddresses:      make([]string, 0),
 		VolumeGrpcAddresses: make([]string, 0),
 		S3GrpcAddresses:     make([]string, 0),
 		Metadata: map[string]string{
@@ -767,6 +767,9 @@ func (s *AdminServer) buildDefaultPluginClusterContext() *plugin_pb.ClusterConte
 		clusterContext.MasterGrpcAddresses = append(clusterContext.MasterGrpcAddresses, masterAddress)
 	}
 
+	// Master returns filers in pb.ServerAddress form (host:httpPort.grpcPort).
+	// Forward that verbatim; each worker converts to a gRPC or HTTP address as
+	// it needs (dialing wants gRPC, the admin shell wants the ServerAddress).
 	filerSeen := map[string]struct{}{}
 	for _, filer := range s.GetAllFilers() {
 		filer = strings.TrimSpace(filer)
@@ -777,7 +780,7 @@ func (s *AdminServer) buildDefaultPluginClusterContext() *plugin_pb.ClusterConte
 			continue
 		}
 		filerSeen[filer] = struct{}{}
-		clusterContext.FilerGrpcAddresses = append(clusterContext.FilerGrpcAddresses, filer)
+		clusterContext.FilerAddresses = append(clusterContext.FilerAddresses, filer)
 	}
 
 	volumeSeen := map[string]struct{}{}
@@ -825,7 +828,7 @@ func (s *AdminServer) buildDefaultPluginClusterContext() *plugin_pb.ClusterConte
 	}
 
 	sort.Strings(clusterContext.MasterGrpcAddresses)
-	sort.Strings(clusterContext.FilerGrpcAddresses)
+	sort.Strings(clusterContext.FilerAddresses)
 	sort.Strings(clusterContext.VolumeGrpcAddresses)
 	sort.Strings(clusterContext.S3GrpcAddresses)
 

weed/admin/dash/worker_grpc_server.go

@@ -2,6 +2,7 @@ package dash
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -15,9 +16,12 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
+	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/status"
 )
 
 const (
@@ -230,6 +234,7 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
 	}
 	s.connections[workerID] = conn
 	s.connMutex.Unlock()
+	stats_collect.AdminWorkerEventsTotal.WithLabelValues("registered").Inc()
 
 	// Register worker with maintenance manager
 	s.registerWorkerWithManager(conn)
@@ -262,23 +267,27 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
 		select {
 		case <-ctx.Done():
 			glog.Infof("Worker %s connection closed: %v", workerID, ctx.Err())
-			s.unregisterWorker(conn)
+			s.unregisterWorker(conn, "unregistered")
 			return nil
 		case <-connCtx.Done():
 			glog.Infof("Worker %s connection cancelled", workerID)
-			s.unregisterWorker(conn)
+			s.unregisterWorker(conn, "unregistered")
 			return nil
 		default:
 		}
 
 		msg, err := stream.Recv()
 		if err != nil {
-			if err == io.EOF {
+			switch {
+			case err == io.EOF:
 				glog.Infof("Worker %s disconnected", workerID)
-			} else {
+			case errors.Is(err, context.Canceled), status.Code(err) == codes.Canceled:
+				// Graceful shutdown on either side cancels the stream.
+				glog.V(1).Infof("Worker %s stream canceled: %v", workerID, err)
+			default:
 				glog.Errorf("Error receiving from worker %s: %v", workerID, err)
 			}
-			s.unregisterWorker(conn)
+			s.unregisterWorker(conn, "unregistered")
 			return err
 		}
 
@@ -331,7 +340,7 @@ func (s *WorkerGrpcServer) handleWorkerMessage(conn *WorkerConnection, msg *work
 
 	case *worker_pb.WorkerMessage_Shutdown:
 		glog.Infof("Worker %s shutting down: %s", workerID, m.Shutdown.Reason)
-		s.unregisterWorker(conn)
+		s.unregisterWorker(conn, "unregistered")
 
 	default:
 		glog.Warningf("Unknown message type from worker %s", workerID)
@@ -598,7 +607,7 @@ func (s *WorkerGrpcServer) safeCloseOutgoingChannel(conn *WorkerConnection, sour
 }
 
 // unregisterWorker removes a worker connection
-func (s *WorkerGrpcServer) unregisterWorker(conn *WorkerConnection) {
+func (s *WorkerGrpcServer) unregisterWorker(conn *WorkerConnection, event string) {
 	s.connMutex.Lock()
 	existingConn, exists := s.connections[conn.workerID]
 	if !exists {
@@ -617,6 +626,7 @@ func (s *WorkerGrpcServer) unregisterWorker(conn *WorkerConnection) {
 	// Remove from map first to prevent duplicate cleanup attempts
 	delete(s.connections, conn.workerID)
 	s.connMutex.Unlock()
+	stats_collect.AdminWorkerEventsTotal.WithLabelValues(event).Inc()
 
 	// Cancel context to signal goroutines to stop
 	conn.cancel()
@@ -658,7 +668,7 @@ func (s *WorkerGrpcServer) cleanupStaleConnections() {
 
 	for _, conn := range toRemove {
 		glog.Warningf("Cleaning up stale worker connection: %s", conn.workerID)
-		s.unregisterWorker(conn)
+		s.unregisterWorker(conn, "stale_removed")
 	}
 }
 

weed/admin/handlers/file_browser_grpc.go

@@ -0,0 +1,291 @@
+package handlers
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"path"
+	"strconv"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// Admin file browser upload chunk sizing — kept in sync with the values
+// s3api uses so files end up split into the same fid-sized pieces the rest of
+// the cluster expects.
+const (
+	adminUploadChunkSize      = 8 * 1024 * 1024
+	adminUploadSmallFileLimit = 256 * 1024
+)
+
+// File browser handlers backed by the filer gRPC service. They bypass the
+// filer's HTTP listener so the UI keeps working when the filer is started
+// with -disableHttp=true; chunk bytes still flow through the volume server
+// HTTP endpoints (which run on their own ports).
+
+// fetchFileContentGrpc reads file content via the filer gRPC service, looking
+// the entry up and then streaming the chunks straight from the volume servers.
+// When maxBytes > 0 the stream is truncated to that many bytes — used by the
+// "is this text?" sniff so unknown-MIME files don't get fully downloaded.
+func (h *FileBrowserHandlers) fetchFileContentGrpc(ctx context.Context, filePath string, maxBytes int) (string, error) {
+	cleanFilePath, err := h.validateAndCleanFilePath(filePath)
+	if err != nil {
+		return "", err
+	}
+
+	entry, err := h.lookupEntry(ctx, cleanFilePath)
+	if err != nil {
+		return "", err
+	}
+	if entry.IsDirectory {
+		return "", fmt.Errorf("path is a directory")
+	}
+
+	size := int64(filer.FileSize(entry))
+	streamSize := size
+	if maxBytes > 0 && streamSize > int64(maxBytes) {
+		streamSize = int64(maxBytes)
+	}
+
+	var buf bytes.Buffer
+	if err := h.streamEntryContent(ctx, entry, streamSize, &buf); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+// downloadFileGrpc streams a file via gRPC + volume server HTTP. The
+// response writer receives the canonical attachment headers and the raw
+// bytes; this replaces the HTTP-to-filer proxy that used to run in
+// DownloadFile.
+func (h *FileBrowserHandlers) downloadFileGrpc(ctx context.Context, filePath string, w http.ResponseWriter) error {
+	cleanFilePath, err := h.validateAndCleanFilePath(filePath)
+	if err != nil {
+		return err
+	}
+
+	entry, err := h.lookupEntry(ctx, cleanFilePath)
+	if err != nil {
+		return err
+	}
+	if entry.IsDirectory {
+		return fmt.Errorf("path is a directory")
+	}
+
+	size := int64(filer.FileSize(entry))
+
+	fileName := path.Base(cleanFilePath)
+	w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": fileName}))
+
+	contentType := ""
+	if entry.Attributes != nil {
+		contentType = entry.Attributes.Mime
+	}
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	w.Header().Set("Content-Type", contentType)
+	w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
+	w.WriteHeader(http.StatusOK)
+
+	return h.streamEntryContent(ctx, entry, size, w)
+}
+
+// uploadFileGrpc streams the upload to volume servers in 8 MiB chunks via the
+// shared chunked-upload helper, then registers the assembled entry through the
+// filer gRPC service. Bytes never enter the admin process's heap as a whole —
+// each chunk is sized to adminUploadChunkSize. Small files (< 256 KiB) are
+// stored inline on the entry, matching the S3 server's behaviour.
+func (h *FileBrowserHandlers) uploadFileGrpc(ctx context.Context, filePath string, fileName string, mimeType string, reader io.Reader) error {
+	cleanFilePath, err := h.validateAndCleanFilePath(filePath)
+	if err != nil {
+		return err
+	}
+	dir := path.Dir(cleanFilePath)
+	if dir == "." {
+		dir = "/"
+	}
+	entryName := path.Base(cleanFilePath)
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+
+	assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*operation.VolumeAssignRequest, *operation.AssignResult, error) {
+		var assignResp *filer_pb.AssignVolumeResponse
+		err := h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+			resp, assignErr := client.AssignVolume(ctx, &filer_pb.AssignVolumeRequest{
+				Count:            int32(count),
+				Path:             cleanFilePath,
+				ExpectedDataSize: expectedDataSize,
+			})
+			if assignErr != nil {
+				return assignErr
+			}
+			if resp.Error != "" {
+				return fmt.Errorf("%s", resp.Error)
+			}
+			assignResp = resp
+			return nil
+		})
+		if err != nil {
+			return nil, nil, err
+		}
+		if assignResp.Location == nil || assignResp.FileId == "" {
+			return nil, nil, fmt.Errorf("assign volume returned empty location")
+		}
+		return nil, &operation.AssignResult{
+			Fid:       assignResp.FileId,
+			Url:       assignResp.Location.Url,
+			PublicUrl: assignResp.Location.PublicUrl,
+			Count:     uint64(count),
+			Auth:      security.EncodedJwt(assignResp.Auth),
+		}, nil
+	}
+
+	chunkResult, err := operation.UploadReaderInChunks(ctx, reader, &operation.ChunkedUploadOption{
+		ChunkSize:       adminUploadChunkSize,
+		SmallFileLimit:  adminUploadSmallFileLimit,
+		SaveSmallInline: true,
+		MimeType:        mimeType,
+		AssignFunc:      assignFunc,
+	})
+	if err != nil {
+		// Partial chunks come back even on error so we can clean them up rather
+		// than leaving orphaned data on volume servers.
+		if chunkResult != nil && len(chunkResult.FileChunks) > 0 {
+			h.deleteOrphanedChunks(chunkResult.FileChunks)
+		}
+		return fmt.Errorf("upload: %w", err)
+	}
+
+	now := time.Now()
+	entry := &filer_pb.Entry{
+		Name: entryName,
+		Attributes: &filer_pb.FuseAttributes{
+			FileSize: uint64(chunkResult.TotalSize),
+			Mtime:    now.Unix(),
+			Crtime:   now.Unix(),
+			FileMode: 0644,
+			Mime:     mimeType,
+		},
+	}
+	if len(chunkResult.SmallContent) > 0 {
+		entry.Content = chunkResult.SmallContent
+	} else {
+		entry.Chunks = chunkResult.FileChunks
+	}
+
+	err = h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		_, createErr := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{
+			Directory: dir,
+			Entry:     entry,
+		})
+		return createErr
+	})
+	if err != nil {
+		if len(chunkResult.FileChunks) > 0 {
+			h.deleteOrphanedChunks(chunkResult.FileChunks)
+		}
+		return fmt.Errorf("create entry: %w", err)
+	}
+	return nil
+}
+
+// deleteOrphanedChunks best-effort removes the chunk fids when an upload
+// fails partway through. Errors are logged; we can't surface them past the
+// caller's primary failure.
+func (h *FileBrowserHandlers) deleteOrphanedChunks(chunks []*filer_pb.FileChunk) {
+	fileIds := make([]string, 0, len(chunks))
+	for _, c := range chunks {
+		if fid := c.GetFileIdString(); fid != "" {
+			fileIds = append(fileIds, fid)
+		}
+	}
+	if len(fileIds) == 0 {
+		return
+	}
+	master := h.adminServer.GetMasterClient()
+	results := operation.DeleteFileIds(master.GetMaster, false, h.adminServer.GetGrpcDialOption(), fileIds)
+	for _, r := range results {
+		if r.Error != "" {
+			glog.Warningf("admin file browser: orphan chunk %s cleanup: %s", r.FileId, r.Error)
+		}
+	}
+}
+
+func (h *FileBrowserHandlers) lookupEntry(ctx context.Context, cleanFilePath string) (*filer_pb.Entry, error) {
+	dir := path.Dir(cleanFilePath)
+	if dir == "." {
+		dir = "/"
+	}
+	name := path.Base(cleanFilePath)
+	var entry *filer_pb.Entry
+	err := h.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		resp, lookupErr := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
+			Directory: dir,
+			Name:      name,
+		})
+		if lookupErr != nil {
+			return lookupErr
+		}
+		if resp.Entry == nil {
+			return fmt.Errorf("not found")
+		}
+		entry = resp.Entry
+		return nil
+	})
+	return entry, err
+}
+
+func (h *FileBrowserHandlers) streamEntryContent(ctx context.Context, entry *filer_pb.Entry, size int64, w io.Writer) error {
+	if size == 0 {
+		// Inline content (small files stored directly on the entry) skip the
+		// chunk pipeline entirely.
+		if len(entry.Content) > 0 {
+			_, err := w.Write(entry.Content)
+			return err
+		}
+		return nil
+	}
+	if len(entry.Content) > 0 && len(entry.GetChunks()) == 0 {
+		_, err := w.Write(entry.Content)
+		return err
+	}
+
+	streamFn, err := filer.PrepareStreamContentWithThrottler(
+		ctx,
+		h.adminServer.GetMasterClient(),
+		volumeServerReadJwt,
+		entry.GetChunks(),
+		0,
+		size,
+		0,
+	)
+	if err != nil {
+		return fmt.Errorf("prepare stream: %w", err)
+	}
+	return streamFn(w)
+}
+
+// volumeServerReadJwt mints a per-fileId Bearer token for reads against a
+// volume server when jwt.signing.read.key is configured. The volume servers
+// are unaware of jwt.filer_signing.read.key — that one only gates the filer
+// HTTP surface, which this code path doesn't touch.
+func volumeServerReadJwt(fileId string) string {
+	v := util.GetViper()
+	signingKey := security.SigningKey(v.GetString("jwt.signing.read.key"))
+	if len(signingKey) == 0 {
+		return ""
+	}
+	expiresAfterSec := v.GetInt("jwt.signing.read.expires_after_seconds")
+	return string(security.GenJwtForVolumeServer(signingKey, expiresAfterSec, fileId))
+}

weed/admin/handlers/file_browser_handlers.go

@@ -1,15 +1,10 @@
 package handlers
 
 import (
-	"bytes"
 	"context"
 	"fmt"
-	"io"
-	"mime"
 	"mime/multipart"
-	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"path"
 	"path/filepath"
@@ -21,9 +16,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
 	"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/http/client"
 )
@@ -49,15 +42,6 @@ func NewFileBrowserHandlers(adminServer *dash.AdminServer) *FileBrowserHandlers
 	}
 }
 
-// newClientWithTimeout creates a temporary http.Client with the specified timeout,
-// reusing the TLS transport from the shared httpClient.
-func (h *FileBrowserHandlers) newClientWithTimeout(timeout time.Duration) http.Client {
-	return http.Client{
-		Transport: h.httpClient.Client.Transport,
-		Timeout:   timeout,
-	}
-}
-
 // ShowFileBrowser renders the file browser page
 func (h *FileBrowserHandlers) ShowFileBrowser(w http.ResponseWriter, r *http.Request) {
 	// Get path from query parameter, default to root
@@ -309,7 +293,7 @@ func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request)
 		}
 
 		// Upload file to filer
-		err = h.uploadFileToFiler(fullPath, fileHeader)
+		err = h.uploadFileToFiler(r.Context(), fullPath, fileHeader)
 
 		if err != nil {
 			failedUploads = append(failedUploads, fmt.Sprintf("%s: %v", fileName, err))
@@ -346,139 +330,21 @@ func (h *FileBrowserHandlers) UploadFile(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// uploadFileToFiler uploads a file directly to the filer using multipart form data
-func (h *FileBrowserHandlers) uploadFileToFiler(filePath string, fileHeader *multipart.FileHeader) error {
-	// Get filer address from admin server
-	filerAddress := h.adminServer.GetFilerAddress()
-	if filerAddress == "" {
-		return fmt.Errorf("filer address not configured")
-	}
-
-	// Validate and sanitize the filer address
-	if err := h.validateFilerAddress(filerAddress); err != nil {
-		return fmt.Errorf("invalid filer address: %w", err)
-	}
-	filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress()
-
-	// Validate and sanitize the file path
-	cleanFilePath, err := h.validateAndCleanFilePath(filePath)
-	if err != nil {
-		return fmt.Errorf("invalid file path: %w", err)
-	}
-
-	// Open the file
+// uploadFileToFiler uploads a file to the cluster via filer gRPC + volume
+// HTTP. This works whether or not the filer is running with -disableHttp=true,
+// since the bytes never traverse the filer's HTTP listener. The multipart
+// file is streamed through the chunked uploader, so the admin process never
+// buffers the entire payload in memory. The caller passes the request
+// context so a client disconnect cancels the in-flight chunk uploads instead
+// of letting them run to completion against the volume servers.
+func (h *FileBrowserHandlers) uploadFileToFiler(ctx context.Context, filePath string, fileHeader *multipart.FileHeader) error {
 	file, err := fileHeader.Open()
 	if err != nil {
 		return fmt.Errorf("failed to open file: %w", err)
 	}
 	defer file.Close()
 
-	// Create multipart form data
-	var body bytes.Buffer
-	writer := multipart.NewWriter(&body)
-
-	// Create form file field with normalized base filename
-	// Use path.Base (not filepath.Base) since cleanFilePath uses URL path semantics
-	baseFileName := path.Base(cleanFilePath)
-	part, err := writer.CreateFormFile("file", baseFileName)
-	if err != nil {
-		return fmt.Errorf("failed to create form file: %w", err)
-	}
-
-	// Copy file content to form
-	_, err = io.Copy(part, file)
-	if err != nil {
-		return fmt.Errorf("failed to copy file content: %w", err)
-	}
-
-	// Close the writer to finalize the form
-	err = writer.Close()
-	if err != nil {
-		return fmt.Errorf("failed to close multipart writer: %w", err)
-	}
-
-	// Create the upload URL - the httpClient will normalize to the correct scheme (http/https)
-	// based on the https.client configuration in security.toml
-	uploadURL := filerFileURL(filerHttpAddress, cleanFilePath)
-
-	// Normalize the URL scheme based on TLS configuration
-	uploadURL, err = h.httpClient.NormalizeHttpScheme(uploadURL)
-	if err != nil {
-		return fmt.Errorf("failed to normalize URL scheme: %w", err)
-	}
-
-	// Create HTTP request
-	req, err := http.NewRequest("POST", uploadURL, &body)
-	if err != nil {
-		return fmt.Errorf("failed to create request: %w", err)
-	}
-
-	// Set content type with boundary
-	req.Header.Set("Content-Type", writer.FormDataContentType())
-
-	// Add JWT Token to Authorization Header
-	h.setupFilerJwtAuth(req, "jwt.filer_signing.key", "jwt.filer_signing.expires_after_seconds", "filer upload")
-
-	// Send request using TLS-aware HTTP client with 60s timeout for large file uploads
-	// lgtm[go/ssrf]
-	// Safe: filerAddress validated by validateFilerAddress() to match configured filer
-	// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
-	client := h.newClientWithTimeout(60 * time.Second)
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("failed to upload file: %w", err)
-	}
-	defer resp.Body.Close()
-
-	// Check response
-	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
-		responseBody, _ := io.ReadAll(resp.Body)
-		return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(responseBody))
-	}
-
-	return nil
-}
-
-// validateFilerAddress validates that the filer address is safe to use
-func (h *FileBrowserHandlers) validateFilerAddress(address string) error {
-	if address == "" {
-		return fmt.Errorf("filer address cannot be empty")
-	}
-
-	// CRITICAL: Only allow the configured filer address to prevent SSRF
-	configuredFiler := h.adminServer.GetFilerAddress()
-	normalizedAddress := pb.ServerAddress(address).ToHttpAddress()
-	normalizedConfigured := pb.ServerAddress(configuredFiler).ToHttpAddress()
-	if normalizedAddress != normalizedConfigured {
-		return fmt.Errorf("address does not match configured filer: got %s, expected %s", address, configuredFiler)
-	}
-
-	// Parse the normalized HTTP address to validate it's a proper host:port format.
-	host, port, err := net.SplitHostPort(normalizedAddress)
-	if err != nil {
-		return fmt.Errorf("invalid address format: %w", err)
-	}
-
-	// Validate host is not empty
-	if host == "" {
-		return fmt.Errorf("host cannot be empty")
-	}
-
-	// Validate port is numeric and in valid range
-	if port == "" {
-		return fmt.Errorf("port cannot be empty")
-	}
-
-	portNum, err := strconv.Atoi(port)
-	if err != nil {
-		return fmt.Errorf("invalid port number: %w", err)
-	}
-
-	if portNum < 1 || portNum > 65535 {
-		return fmt.Errorf("port number must be between 1 and 65535")
-	}
-
-	return nil
+	return h.uploadFileGrpc(ctx, filePath, fileHeader.Filename, fileHeader.Header.Get("Content-Type"), file)
 }
 
 // validateAndCleanFilePath validates and cleans the file path to prevent path traversal
@@ -507,161 +373,56 @@ func (h *FileBrowserHandlers) validateAndCleanFilePath(filePath string) (string,
 	return cleanPath, nil
 }
 
-// filerFileURL joins the filer HTTP address with a validated file path, URL-escaping
-// the path so that control characters and other bytes that are legal in S3 object keys
-// cannot inject into the HTTP request target.
-func filerFileURL(filerHttpAddress, cleanFilePath string) string {
-	return filerHttpAddress + (&url.URL{Path: cleanFilePath}).EscapedPath()
-}
-
-// fetchFileContent fetches file content from the filer and returns the content or an error.
+// fetchFileContent fetches file content via the filer gRPC service. It is
+// used for the "view as text" path, so the maxBytes cap matches the 1 MB
+// limit the caller already applies before invoking us.
 func (h *FileBrowserHandlers) fetchFileContent(filePath string, timeout time.Duration) (string, error) {
-	filerAddress := h.adminServer.GetFilerAddress()
-	if filerAddress == "" {
-		return "", fmt.Errorf("filer address not configured")
-	}
-
-	if err := h.validateFilerAddress(filerAddress); err != nil {
-		return "", fmt.Errorf("invalid filer address configuration: %w", err)
-	}
-	filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress()
-
-	cleanFilePath, err := h.validateAndCleanFilePath(filePath)
-	if err != nil {
-		return "", err
-	}
-
-	// Create the file URL with proper scheme based on TLS configuration
-	fileURL := filerFileURL(filerHttpAddress, cleanFilePath)
-	fileURL, err = h.httpClient.NormalizeHttpScheme(fileURL)
-	if err != nil {
-		return "", fmt.Errorf("failed to construct file URL: %w", err)
-	}
-
-	// lgtm[go/ssrf]
-	// Safe: filerAddress validated by validateFilerAddress() to match configured filer
-	// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
-	client := h.newClientWithTimeout(timeout)
-	req, err := http.NewRequest("GET", fileURL, nil)
-	if err != nil {
-		return "", fmt.Errorf("failed to create request: %w", err)
-	}
-	h.addFilerJwtAuthHeader(req)
-	resp, err := client.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("failed to fetch file from filer: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return "", fmt.Errorf("filer returned status %d but failed to read response body: %w", resp.StatusCode, err)
-		}
-		return "", fmt.Errorf("filer returned status %d: %s", resp.StatusCode, string(body))
-	}
-
-	contentBytes, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", fmt.Errorf("failed to read file content: %w", err)
-	}
-
-	return string(contentBytes), nil
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	return h.fetchFileContentGrpc(ctx, filePath, 0)
 }
 
-// DownloadFile handles file download requests by proxying through the Admin UI server
-// This ensures mTLS works correctly since the Admin UI server has the client certificates
+// DownloadFile streams a file straight from the volume servers via the filer
+// gRPC service, so the admin file browser keeps working even when the filer
+// is started with -disableHttp=true.
 func (h *FileBrowserHandlers) DownloadFile(w http.ResponseWriter, r *http.Request) {
 	filePath := r.URL.Query().Get("path")
 	if filePath == "" {
 		writeJSONError(w, http.StatusBadRequest, "File path is required")
 		return
 	}
-
-	// Get filer address
-	filerAddress := h.adminServer.GetFilerAddress()
-	if filerAddress == "" {
-		writeJSONError(w, http.StatusInternalServerError, "Filer address not configured")
-		return
-	}
-
-	// Validate filer address to prevent SSRF
-	if err := h.validateFilerAddress(filerAddress); err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "Invalid filer address configuration")
-		return
-	}
-	filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress()
-
-	// Validate and sanitize the file path
-	cleanFilePath, err := h.validateAndCleanFilePath(filePath)
-	if err != nil {
-		writeJSONError(w, http.StatusBadRequest, "Invalid file path: "+err.Error())
-		return
-	}
-
-	// Create the download URL with proper scheme based on TLS configuration
-	downloadURL := filerFileURL(filerHttpAddress, cleanFilePath)
-	downloadURL, err = h.httpClient.NormalizeHttpScheme(downloadURL)
-	if err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "Failed to construct download URL: "+err.Error())
-		return
-	}
-
-	// Proxy the download through the Admin UI server to support mTLS
-	// lgtm[go/ssrf]
-	// Safe: filerAddress validated by validateFilerAddress() to match configured filer
-	// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
-	// Use request context so download is cancelled when client disconnects
-	req, err := http.NewRequestWithContext(r.Context(), "GET", downloadURL, nil)
-	if err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "Failed to create request: "+err.Error())
-		return
-	}
-	client := h.newClientWithTimeout(5 * time.Minute) // Longer timeout for large file downloads
-
-	h.addFilerJwtAuthHeader(req)
-
-	resp, err := client.Do(req)
-	if err != nil {
-		writeJSONError(w, http.StatusBadGateway, "Failed to fetch file from filer: "+err.Error())
-		return
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			writeJSONError(w, resp.StatusCode, fmt.Sprintf("Filer returned status %d but failed to read response body: %v", resp.StatusCode, err))
+	tracker := &responseWriteTracker{ResponseWriter: w}
+	if err := h.downloadFileGrpc(r.Context(), filePath, tracker); err != nil {
+		// Once bytes have been written we can't switch to a JSON error body
+		// without corrupting the partial response — log and stop. Before any
+		// write the response is still uncommitted, so a 502 with details is
+		// safe.
+		if tracker.committed {
+			glog.Errorf("Error streaming file download: %v", err)
 			return
 		}
-		writeJSONError(w, resp.StatusCode, fmt.Sprintf("Filer returned status %d: %s", resp.StatusCode, string(body)))
-		return
+		writeJSONError(w, http.StatusBadGateway, "Failed to fetch file: "+err.Error())
 	}
+}
 
-	// Set headers for file download
-	fileName := filepath.Base(cleanFilePath)
-	// Use mime.FormatMediaType for RFC 6266 compliant Content-Disposition,
-	// properly handling non-ASCII characters and special characters
-	w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": fileName}))
+// responseWriteTracker wraps http.ResponseWriter to record whether the
+// response has been committed (status line + headers sent). DownloadFile
+// uses this instead of probing Header() so future header-setting code
+// reorganization can't silently break the "did we already send bytes?"
+// detection.
+type responseWriteTracker struct {
+	http.ResponseWriter
+	committed bool
+}
 
-	// Use content type from filer response, or default to octet-stream
-	contentType := resp.Header.Get("Content-Type")
-	if contentType == "" {
-		contentType = "application/octet-stream"
-	}
-	w.Header().Set("Content-Type", contentType)
+func (t *responseWriteTracker) WriteHeader(code int) {
+	t.committed = true
+	t.ResponseWriter.WriteHeader(code)
+}
 
-	// Set content length if available
-	if resp.ContentLength > 0 {
-		w.Header().Set("Content-Length", fmt.Sprintf("%d", resp.ContentLength))
-	}
-
-	// Stream the response body to the client
-	w.WriteHeader(http.StatusOK)
-	_, err = io.Copy(w, resp.Body)
-	if err != nil {
-		glog.Errorf("Error streaming file download: %v", err)
-	}
+func (t *responseWriteTracker) Write(p []byte) (int, error) {
+	t.committed = true
+	return t.ResponseWriter.Write(p)
 }
 
 // ViewFile handles file viewing requests (for text files, images, etc.)
@@ -883,64 +644,16 @@ func (h *FileBrowserHandlers) formatBytes(bytes int64) string {
 
 // Helper function to check if a file is likely a text file by checking content
 func (h *FileBrowserHandlers) isLikelyTextFile(filePath string, maxCheckSize int64) bool {
-	filerAddress := h.adminServer.GetFilerAddress()
-	if filerAddress == "" {
-		return false
-	}
-
-	// Validate filer address to prevent SSRF
-	if err := h.validateFilerAddress(filerAddress); err != nil {
-		glog.Errorf("Invalid filer address: %v", err)
-		return false
-	}
-	filerHttpAddress := pb.ServerAddress(filerAddress).ToHttpAddress()
-
-	cleanFilePath, err := h.validateAndCleanFilePath(filePath)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	content, err := h.fetchFileContentGrpc(ctx, filePath, int(maxCheckSize))
 	if err != nil {
 		return false
 	}
-
-	// Create the file URL with proper scheme based on TLS configuration
-	fileURL := filerFileURL(filerHttpAddress, cleanFilePath)
-	fileURL, err = h.httpClient.NormalizeHttpScheme(fileURL)
-	if err != nil {
-		glog.Errorf("Failed to normalize URL scheme: %v", err)
-		return false
+	if len(content) == 0 {
+		return true
 	}
-
-	// lgtm[go/ssrf]
-	// Safe: filerAddress validated by validateFilerAddress() to match configured filer
-	// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
-	client := h.newClientWithTimeout(10 * time.Second)
-	req, err := http.NewRequest("GET", fileURL, nil)
-	if err != nil {
-		glog.Errorf("Failed to create request: %v", err)
-		return false
-	}
-	h.addFilerJwtAuthHeader(req)
-	resp, err := client.Do(req)
-	if err != nil {
-		return false
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return false
-	}
-
-	// Read first few bytes to check if it's text
-	buffer := make([]byte, min(maxCheckSize, 512))
-	n, err := resp.Body.Read(buffer)
-	if err != nil && err != io.EOF {
-		return false
-	}
-
-	if n == 0 {
-		return true // Empty file can be considered text
-	}
-
-	// Check if content is printable text
-	return h.isPrintableText(buffer[:n])
+	return h.isPrintableText([]byte(content))
 }
 
 // Helper function to check if content is printable text
@@ -973,35 +686,3 @@ func min(a, b int64) int64 {
 	return b
 }
 
-// setupFilerJwtAuth generates a JWT token and adds it to the request Authorization header if configured.
-func (h *FileBrowserHandlers) setupFilerJwtAuth(req *http.Request, keyPath, expiresPath, operation string) {
-	// Load security configuration
-	v := util.GetViper()
-
-	// Read Filer JWT token from security.toml
-	signingKey := security.SigningKey(v.GetString(keyPath))
-	expiresAfterSec := v.GetInt(expiresPath)
-
-	//  Generate JWT token to authenticate with Filer
-	var jwtToken security.EncodedJwt
-	if len(signingKey) > 0 {
-		jwtToken = security.GenJwtForFilerServer(signingKey, expiresAfterSec)
-		glog.V(4).Infof("Generated JWT token for %s (expires in %d sec)", operation, expiresAfterSec)
-	} else {
-		if v.GetString("jwt.signing.key") != "" {
-			glog.Warningf("JWT %s key not configured, but general JWT security is enabled. %s without authentication.", keyPath, operation)
-		} else {
-			glog.V(1).Infof("No JWT signing key configured, %s without authentication", operation)
-		}
-	}
-
-	// Add JWT Token to Authorization Header
-	if jwtToken != "" {
-		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(jwtToken)))
-		glog.V(4).Infof("Added JWT authorization header for %s", operation)
-	}
-}
-
-func (h *FileBrowserHandlers) addFilerJwtAuthHeader(req *http.Request) {
-	h.setupFilerJwtAuth(req, "jwt.filer_signing.read.key", "jwt.filer_signing.read.expires_after_seconds", "filer request")
-}

weed/admin/handlers/file_browser_handlers_test.go

@@ -42,21 +42,3 @@ func TestValidateAndCleanFilePath_RejectsEmpty(t *testing.T) {
 	}
 }
 
-func TestFilerFileURL_EscapesControlChars(t *testing.T) {
-	cases := []struct {
-		addr string
-		path string
-		want string
-	}{
-		{"http://127.0.0.1:8888", "/buckets/profilebuilder/3testGB.zip\n ", "http://127.0.0.1:8888/buckets/profilebuilder/3testGB.zip%0A%20"},
-		{"http://127.0.0.1:8888", "/buckets/profilebuilder/file\rname", "http://127.0.0.1:8888/buckets/profilebuilder/file%0Dname"},
-		{"http://127.0.0.1:8888", "/buckets/profilebuilder/file\x00name", "http://127.0.0.1:8888/buckets/profilebuilder/file%00name"},
-		// Plain path round-trips unchanged.
-		{"http://h:1", "/a/b.txt", "http://h:1/a/b.txt"},
-	}
-	for _, tc := range cases {
-		if got := filerFileURL(tc.addr, tc.path); got != tc.want {
-			t.Errorf("filerFileURL(%q, %q) = %q, want %q", tc.addr, tc.path, got, tc.want)
-		}
-	}
-}

weed/admin/maintenance/maintenance_integration.go

@@ -26,6 +26,10 @@ type MaintenanceIntegration struct {
 	// Active topology for task detection and target selection
 	activeTopology *topology.ActiveTopology
 
+	// Master's default replication, refreshed by the scanner each cycle and
+	// passed to detectors as the replica-placement fallback (matches the shell).
+	defaultReplicaPlacement string
+
 	// Type conversion maps
 	taskTypeMap    map[types.TaskType]MaintenanceTaskType
 	revTaskTypeMap map[MaintenanceTaskType]types.TaskType
@@ -219,9 +223,10 @@ func (s *MaintenanceIntegration) ScanWithTaskDetectors(volumeMetrics []*types.Vo
 
 	// Create cluster info
 	clusterInfo := &types.ClusterInfo{
-		TotalVolumes:   len(filteredMetrics),
-		LastUpdated:    time.Now(),
-		ActiveTopology: s.activeTopology, // Provide ActiveTopology for destination planning
+		TotalVolumes:            len(filteredMetrics),
+		LastUpdated:             time.Now(),
+		ActiveTopology:          s.activeTopology, // Provide ActiveTopology for destination planning
+		DefaultReplicaPlacement: s.defaultReplicaPlacement,
 	}
 
 	// Run detection for each registered task type
@@ -271,6 +276,12 @@ func (s *MaintenanceIntegration) ScanWithTaskDetectors(volumeMetrics []*types.Vo
 	return allResults, nil
 }
 
+// SetDefaultReplicaPlacement records the master's default replication so detectors
+// can use it as the replica-placement fallback (matching the shell).
+func (s *MaintenanceIntegration) SetDefaultReplicaPlacement(replicaPlacement string) {
+	s.defaultReplicaPlacement = replicaPlacement
+}
+
 // UpdateTopologyInfo updates the volume shard tracker with topology information for empty servers
 func (s *MaintenanceIntegration) UpdateTopologyInfo(topologyInfo *master_pb.TopologyInfo) error {
 	// Log topology details before update for diagnostics

weed/admin/maintenance/maintenance_manager.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
 	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
@@ -315,6 +316,7 @@ func (mm *MaintenanceManager) performScan() {
 	glog.Infof("Starting maintenance scan...")
 
 	results, err := mm.scanner.ScanForMaintenanceTasks()
+	stats_collect.AdminMaintenanceLastScanTimestampSeconds.SetToCurrentTime()
 	if err != nil {
 		// Handle scan error
 		mm.mutex.Lock()
@@ -518,6 +520,11 @@ func (mm *MaintenanceManager) GetWorkers() []*MaintenanceWorker {
 	return mm.queue.GetWorkers()
 }
 
+// GetWorkerSlotTotals returns worker count and aggregate used/max task slots.
+func (mm *MaintenanceManager) GetWorkerSlotTotals() (workers, used, max int) {
+	return mm.queue.GetWorkerSlotTotals()
+}
+
 // TriggerScan manually triggers a maintenance scan
 func (mm *MaintenanceManager) TriggerScan() error {
 	return mm.triggerScanInternal(true)

weed/admin/maintenance/maintenance_queue.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 )
 
 // NewMaintenanceQueue creates a new maintenance queue
@@ -482,12 +483,14 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 
 	// Calculate task duration
 	var duration time.Duration
-	if task.StartedAt != nil {
+	hadStart := task.StartedAt != nil
+	if hadStart {
 		duration = completedTime.Sub(*task.StartedAt)
 	}
 
 	// Capture workerID before it may be cleared during retry
 	originalWorkerID := task.WorkerID
+	taskType := string(task.Type)
 
 	var taskToSave *MaintenanceTask
 	var logFn func()
@@ -577,6 +580,18 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
 	}
 	mq.mutex.Unlock()
 
+	// Record terminal-state metrics. A retry leaves the task pending, so it
+	// is not counted as completed or failed here.
+	switch taskStatus {
+	case TaskStatusCompleted:
+		stats_collect.AdminMaintenanceTasksCompletedTotal.WithLabelValues(taskType, "completed").Inc()
+	case TaskStatusFailed:
+		stats_collect.AdminMaintenanceTasksCompletedTotal.WithLabelValues(taskType, "failed").Inc()
+	}
+	if hadStart && (taskStatus == TaskStatusCompleted || taskStatus == TaskStatusFailed) {
+		stats_collect.AdminMaintenanceTaskDurationSeconds.WithLabelValues(taskType).Observe(duration.Seconds())
+	}
+
 	// Only persist non-terminal tasks (retries). Completed/failed tasks stay
 	// in memory for the UI but are not written to disk — they would just
 	// accumulate and slow down future startups.
@@ -819,6 +834,20 @@ func (mq *MaintenanceQueue) GetWorkers() []*MaintenanceWorker {
 	return workers
 }
 
+// GetWorkerSlotTotals aggregates worker count and used/max task slots under the
+// lock, so callers don't read live worker fields that task updates mutate.
+func (mq *MaintenanceQueue) GetWorkerSlotTotals() (workers, used, max int) {
+	mq.mutex.RLock()
+	defer mq.mutex.RUnlock()
+
+	for _, worker := range mq.workers {
+		workers++
+		used += worker.CurrentLoad
+		max += worker.MaxConcurrent
+	}
+	return
+}
+
 // generateTaskID generates a unique ID for tasks
 func generateTaskID() string {
 	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"

weed/admin/maintenance/maintenance_scanner.go

@@ -51,6 +51,10 @@ func (ms *MaintenanceScanner) ScanForMaintenanceTasks() ([]*TaskDetectionResult,
 			}
 		}
 
+		// Refresh the master's default replication so detectors can use it as the
+		// replica-placement fallback (matches the shell ec.balance default).
+		ms.integration.SetDefaultReplicaPlacement(ms.getDefaultReplicaPlacement())
+
 		// Use task detection system with complete cluster information
 		results, err := ms.integration.ScanWithTaskDetectors(taskMetrics)
 		if err != nil {
@@ -67,6 +71,26 @@ func (ms *MaintenanceScanner) ScanForMaintenanceTasks() ([]*TaskDetectionResult,
 	return []*TaskDetectionResult{}, nil
 }
 
+// getDefaultReplicaPlacement reads the master's configured default replication,
+// used by detectors as the replica-placement fallback. Returns "" on error so
+// detectors fall back to even spread rather than failing the scan.
+func (ms *MaintenanceScanner) getDefaultReplicaPlacement() string {
+	var replicaPlacement string
+	err := ms.adminClient.WithMasterClient(func(client master_pb.SeaweedClient) error {
+		resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		replicaPlacement = resp.DefaultReplication
+		return nil
+	})
+	if err != nil {
+		glog.V(1).Infof("could not fetch master default replication: %v", err)
+		return ""
+	}
+	return replicaPlacement
+}
+
 // getVolumeHealthMetrics collects health information for all volumes.
 // Returns metrics in task-system format directly (no intermediate copy) and
 // the topology info for updating the active topology.

weed/admin/plugin/cluster_rate_limit.go

@@ -113,7 +113,7 @@ func cloneClusterContext(in *plugin_pb.ClusterContext) *plugin_pb.ClusterContext
 	}
 	out := &plugin_pb.ClusterContext{
 		MasterGrpcAddresses: in.MasterGrpcAddresses,
-		FilerGrpcAddresses:  in.FilerGrpcAddresses,
+		FilerAddresses:      in.FilerAddresses,
 		VolumeGrpcAddresses: in.VolumeGrpcAddresses,
 		S3GrpcAddresses:     in.S3GrpcAddresses,
 	}

weed/admin/plugin/job_execution_plan.go

@@ -115,6 +115,7 @@ func buildErasureCodingExecutionPlan(params *worker_pb.TaskParams) map[string]in
 			source.DataCenter,
 			source.Rack,
 			source.VolumeId,
+			source.DiskId,
 			source.ShardIds,
 			dataShards,
 		))
@@ -132,6 +133,7 @@ func buildErasureCodingExecutionPlan(params *worker_pb.TaskParams) map[string]in
 			target.DataCenter,
 			target.Rack,
 			target.VolumeId,
+			target.DiskId,
 			target.ShardIds,
 			dataShards,
 		))
@@ -147,6 +149,7 @@ func buildErasureCodingExecutionPlan(params *worker_pb.TaskParams) map[string]in
 				"target_data_center": strings.TrimSpace(target.DataCenter),
 				"target_rack":        strings.TrimSpace(target.Rack),
 				"target_volume_id":   int(target.VolumeId),
+				"target_disk_id":     int(target.DiskId),
 			})
 		}
 	}
@@ -182,6 +185,7 @@ func buildExecutionEndpoint(
 	dataCenter string,
 	rack string,
 	volumeID uint32,
+	diskID uint32,
 	shardIDs []uint32,
 	dataShardCount int,
 ) map[string]interface{} {
@@ -201,6 +205,7 @@ func buildExecutionEndpoint(
 		"data_center":      strings.TrimSpace(dataCenter),
 		"rack":             strings.TrimSpace(rack),
 		"volume_id":        int(volumeID),
+		"disk_id":          int(diskID),
 		"shard_ids":        allShards,
 		"data_shard_ids":   dataShards,
 		"parity_shard_ids": parityShards,

weed/admin/plugin/plugin_monitor_test.go

@@ -423,6 +423,7 @@ func TestTrackExecutionStartStoresErasureCodingExecutionPlan(t *testing.T) {
 				DataCenter: "dc1",
 				Rack:       "rack1",
 				VolumeId:   29,
+				DiskId:     5,
 			},
 		},
 		Targets: []*worker_pb.TaskTarget{
@@ -431,6 +432,7 @@ func TestTrackExecutionStartStoresErasureCodingExecutionPlan(t *testing.T) {
 				DataCenter: "dc1",
 				Rack:       "rack2",
 				VolumeId:   29,
+				DiskId:     2,
 				ShardIds:   []uint32{0, 10},
 			},
 			{
@@ -438,6 +440,7 @@ func TestTrackExecutionStartStoresErasureCodingExecutionPlan(t *testing.T) {
 				DataCenter: "dc2",
 				Rack:       "rack3",
 				VolumeId:   29,
+				DiskId:     3,
 				ShardIds:   []uint32{1, 11},
 			},
 		},
@@ -486,10 +489,28 @@ func TestTrackExecutionStartStoresErasureCodingExecutionPlan(t *testing.T) {
 	if plan["volume_id"] != float64(29) {
 		t.Fatalf("unexpected execution plan volume id: %+v", plan["volume_id"])
 	}
+	sourcesRaw, ok := plan["sources"].([]interface{})
+	if !ok || len(sourcesRaw) != 1 {
+		t.Fatalf("unexpected sources in execution plan: %+v", plan["sources"])
+	}
+	firstSource, ok := sourcesRaw[0].(map[string]interface{})
+	if !ok {
+		t.Fatalf("unexpected source payload: %+v", sourcesRaw[0])
+	}
+	if firstSource["disk_id"] != float64(5) {
+		t.Fatalf("unexpected source disk_id: %+v", firstSource["disk_id"])
+	}
 	targets, ok := plan["targets"].([]interface{})
 	if !ok || len(targets) != 2 {
 		t.Fatalf("unexpected targets in execution plan: %+v", plan["targets"])
 	}
+	firstTarget, ok := targets[0].(map[string]interface{})
+	if !ok {
+		t.Fatalf("unexpected target payload: %+v", targets[0])
+	}
+	if firstTarget["disk_id"] != float64(2) {
+		t.Fatalf("unexpected target disk_id: %+v", firstTarget["disk_id"])
+	}
 	assignments, ok := plan["shard_assignments"].([]interface{})
 	if !ok || len(assignments) != 4 {
 		t.Fatalf("unexpected shard assignments in execution plan: %+v", plan["shard_assignments"])
@@ -501,6 +522,16 @@ func TestTrackExecutionStartStoresErasureCodingExecutionPlan(t *testing.T) {
 	if firstAssignment["shard_id"] != float64(0) || firstAssignment["kind"] != "data" {
 		t.Fatalf("unexpected first assignment: %+v", firstAssignment)
 	}
+	if firstAssignment["target_disk_id"] != float64(2) {
+		t.Fatalf("unexpected first assignment target_disk_id: %+v", firstAssignment["target_disk_id"])
+	}
+	secondAssignment, ok := assignments[1].(map[string]interface{})
+	if !ok {
+		t.Fatalf("unexpected second assignment payload: %+v", assignments[1])
+	}
+	if secondAssignment["shard_id"] != float64(1) || secondAssignment["target_disk_id"] != float64(3) {
+		t.Fatalf("unexpected second assignment: %+v", secondAssignment)
+	}
 }
 
 func TestBuildJobDetailIncludesActivitiesAndRunRecord(t *testing.T) {

weed/admin/topology/capacity.go

@@ -54,6 +54,70 @@ func (at *ActiveTopology) GetEffectiveAvailableCapacityDetailed(nodeID string, d
 	return at.getEffectiveAvailableCapacityUnsafe(disk)
 }
 
+// GetEffectiveAvailableEcShardSlots returns a disk's free EC shard slots,
+// accounting for in-flight task reservations at shard granularity. Unlike the
+// volume-slot views (GetDisksWithEffectiveCapacity / GetEffectiveAvailableCapacity),
+// this does not truncate sub-volume shard reservations: it subtracts the full
+// reservation impact (volume slots converted to shard slots, plus the raw shard
+// slots) so a reservation that is not a whole multiple of ShardsPerVolumeSlot is
+// not lost. It does NOT subtract the EC shards already persisted on the disk;
+// callers that track those (from EcShardInfos) subtract them separately.
+//
+// shardsPerVolume is the number of EC shards of the target collection that fit in
+// one volume slot (i.e. its data-shard count): a 4+2 volume's shards are ~1/4 of a
+// volume each, so one volume slot holds 4 of them, not the default
+// ShardsPerVolumeSlot. Pass <= 0 to use the default. Using the target ratio keeps
+// Place from over-filling a disk for low-data-shard layouts.
+func (at *ActiveTopology) GetEffectiveAvailableEcShardSlots(nodeID string, diskID uint32, shardsPerVolume int) int {
+	if shardsPerVolume <= 0 {
+		shardsPerVolume = ShardsPerVolumeSlot
+	}
+
+	at.mutex.RLock()
+	defer at.mutex.RUnlock()
+
+	diskKey := fmt.Sprintf("%s:%d", nodeID, diskID)
+	disk, exists := at.disks[diskKey]
+	if !exists || disk.DiskInfo == nil || disk.DiskInfo.DiskInfo == nil {
+		return 0
+	}
+
+	info := disk.DiskInfo.DiskInfo
+	base := info.MaxVolumeCount - info.VolumeCount
+	if base <= 0 && info.MaxVolumeCount == 0 && info.VolumeCount == 0 &&
+		len(info.VolumeInfos) == 0 && len(info.EcShardInfos) == 0 {
+		// Freshly started empty servers can report max=0 before publishing concrete
+		// limits; keep one provisional slot so EC placement still sees the disk,
+		// mirroring getEffectiveAvailableCapacityUnsafe.
+		base = 1
+	}
+	if base < 0 {
+		base = 0
+	}
+	// calculateTaskStorageImpact reports consumption as positive, so subtract it.
+	// Volume-slot reservations scale by the target ratio; the sub-volume shard-slot
+	// remainder is in default units and subtracted as-is (a small approximation).
+	impact := at.getEffectiveCapacityUnsafe(disk)
+	// impact.ShardSlots is recorded in default ShardsPerVolumeSlot units; convert it
+	// to the target ratio's shard slots before subtracting (identity when
+	// shardsPerVolume == ShardsPerVolumeSlot). Round a positive reservation up so a
+	// sub-slot reservation (e.g. 1 default slot against a 4-shard target) is not
+	// truncated to zero and wrongly counted as free.
+	scaledShardImpact := int64(impact.ShardSlots) * int64(shardsPerVolume)
+	if scaledShardImpact > 0 {
+		scaledShardImpact = (scaledShardImpact + int64(ShardsPerVolumeSlot) - 1) / int64(ShardsPerVolumeSlot)
+	} else {
+		scaledShardImpact /= int64(ShardsPerVolumeSlot)
+	}
+	free := base*int64(shardsPerVolume) -
+		int64(impact.VolumeSlots)*int64(shardsPerVolume) -
+		scaledShardImpact
+	if free < 0 {
+		free = 0
+	}
+	return int(free)
+}
+
 // GetEffectiveCapacityImpact returns the StorageSlotChange impact for a disk
 // This shows the net impact from all pending and assigned tasks
 func (at *ActiveTopology) GetEffectiveCapacityImpact(nodeID string, diskID uint32) StorageSlotChange {

weed/admin/topology/count_topology_resources_test.go

@@ -0,0 +1,54 @@
+package topology
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// TestCountTopologyResources_multiDiskPerNode covers the case where the master
+// keys DiskInfos by disk type, so several same-type physical disks on a node
+// collapse into a single DiskInfo entry. Counting len(DiskInfos) under-reports
+// the physical disk count and disagrees with the per-disk activeDisk map that
+// the rest of the admin topology builds via SplitByPhysicalDisk.
+func TestCountTopologyResources_multiDiskPerNode(t *testing.T) {
+	makeNode := func(id string) *master_pb.DataNodeInfo {
+		var ecShardInfos []*master_pb.VolumeEcShardInformationMessage
+		for diskId := uint32(0); diskId < 6; diskId++ {
+			ecShardInfos = append(ecShardInfos, &master_pb.VolumeEcShardInformationMessage{
+				Id:          diskId + 1,
+				DiskId:      diskId,
+				EcIndexBits: 1,
+			})
+		}
+		return &master_pb.DataNodeInfo{
+			Id: id,
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"": {Type: "", MaxVolumeCount: 60, EcShardInfos: ecShardInfos},
+			},
+		}
+	}
+	topo := &master_pb.TopologyInfo{
+		Id: "multi_disk_topo",
+		DataCenterInfos: []*master_pb.DataCenterInfo{{
+			Id: "dc1",
+			RackInfos: []*master_pb.RackInfo{{
+				Id: "rack1",
+				DataNodeInfos: []*master_pb.DataNodeInfo{
+					makeNode("node1"), makeNode("node2"), makeNode("node3"),
+				},
+			}},
+		}},
+	}
+
+	dcCount, nodeCount, diskCount := CountTopologyResources(topo)
+	if dcCount != 1 {
+		t.Errorf("dcCount = %d, want 1", dcCount)
+	}
+	if nodeCount != 3 {
+		t.Errorf("nodeCount = %d, want 3", nodeCount)
+	}
+	if diskCount != 18 {
+		t.Errorf("diskCount = %d, want 18 (6 physical disks x 3 nodes)", diskCount)
+	}
+}

weed/admin/topology/topology_management.go

@@ -9,74 +9,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 )
 
-// splitDiskInfoByPhysicalDisk returns one master_pb.DiskInfo per physical
-// disk_id observed in VolumeInfos / EcShardInfos. Multiple same-type physical
-// disks collapse to one DiskInfo at the master; per-volume/per-shard records
-// keep the original disk_id and are the authoritative signal here. Capacity
-// is split evenly — the wire format doesn't carry per-disk capacity yet.
-func splitDiskInfoByPhysicalDisk(diskInfo *master_pb.DiskInfo) []*master_pb.DiskInfo {
-	if diskInfo == nil {
-		return nil
-	}
-
-	// Records with DiskId=0 and a non-zero outer DiskId belong to the outer
-	// disk — handles older payloads / fixtures that omit the per-record id.
-	normalize := func(id uint32) uint32 {
-		if id == 0 && diskInfo.DiskId != 0 {
-			return diskInfo.DiskId
-		}
-		return id
-	}
-
-	diskIDs := make(map[uint32]struct{})
-	for _, vi := range diskInfo.VolumeInfos {
-		diskIDs[normalize(vi.DiskId)] = struct{}{}
-	}
-	for _, eci := range diskInfo.EcShardInfos {
-		diskIDs[normalize(eci.DiskId)] = struct{}{}
-	}
-	if len(diskIDs) == 0 {
-		diskIDs[diskInfo.DiskId] = struct{}{}
-	}
-
-	if len(diskIDs) == 1 {
-		for diskID := range diskIDs {
-			if diskID == diskInfo.DiskId {
-				return []*master_pb.DiskInfo{diskInfo}
-			}
-		}
-	}
-
-	perDiskVolumes := make(map[uint32][]*master_pb.VolumeInformationMessage)
-	for _, vi := range diskInfo.VolumeInfos {
-		perDiskVolumes[normalize(vi.DiskId)] = append(perDiskVolumes[normalize(vi.DiskId)], vi)
-	}
-	perDiskShards := make(map[uint32][]*master_pb.VolumeEcShardInformationMessage)
-	for _, eci := range diskInfo.EcShardInfos {
-		perDiskShards[normalize(eci.DiskId)] = append(perDiskShards[normalize(eci.DiskId)], eci)
-	}
-
-	count := int64(len(diskIDs))
-	share := func(total int64) int64 { return total / count }
-
-	result := make([]*master_pb.DiskInfo, 0, len(diskIDs))
-	for diskID := range diskIDs {
-		result = append(result, &master_pb.DiskInfo{
-			Type:              diskInfo.Type,
-			MaxVolumeCount:    share(diskInfo.MaxVolumeCount),
-			VolumeCount:       int64(len(perDiskVolumes[diskID])),
-			FreeVolumeCount:   share(diskInfo.FreeVolumeCount),
-			ActiveVolumeCount: share(diskInfo.ActiveVolumeCount),
-			RemoteVolumeCount: share(diskInfo.RemoteVolumeCount),
-			VolumeInfos:       perDiskVolumes[diskID],
-			EcShardInfos:      perDiskShards[diskID],
-			DiskId:            diskID,
-			Tags:              append([]string(nil), diskInfo.Tags...),
-		})
-	}
-	return result
-}
-
 // CountTopologyResources counts datacenters, nodes, and disks in topology info
 func CountTopologyResources(topologyInfo *master_pb.TopologyInfo) (dcCount, nodeCount, diskCount int) {
 	if topologyInfo == nil {
@@ -87,7 +19,12 @@ func CountTopologyResources(topologyInfo *master_pb.TopologyInfo) (dcCount, node
 		for _, rack := range dc.RackInfos {
 			nodeCount += len(rack.DataNodeInfos)
 			for _, node := range rack.DataNodeInfos {
-				diskCount += len(node.DiskInfos)
+				// DiskInfos is keyed by disk type, so same-type physical disks
+				// collapse into one entry. Count physical disks so the number
+				// matches the per-disk activeDisk map.
+				for _, diskInfo := range node.DiskInfos {
+					diskCount += len(diskInfo.SplitByPhysicalDisk())
+				}
 			}
 		}
 	}
@@ -142,10 +79,10 @@ func (at *ActiveTopology) UpdateTopology(topologyInfo *master_pb.TopologyInfo) e
 					disks:      make(map[uint32]*activeDisk),
 				}
 
-				// One activeDisk per physical disk_id (#9369): the master keys
+				// One activeDisk per physical disk_id: the master keys
 				// DiskInfos by disk type, so same-type disks must be split out.
 				for diskType, diskInfo := range nodeInfo.DiskInfos {
-					perDiskInfos := splitDiskInfoByPhysicalDisk(diskInfo)
+					perDiskInfos := diskInfo.SplitByPhysicalDisk()
 					for _, perDisk := range perDiskInfos {
 						disk := &activeDisk{
 							DiskInfo: &DiskInfo{

weed/admin/view/app/admin_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.977
+// templ: version: v0.3.1001
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_brokers_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.977
+// templ: version: v0.3.1001
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_collections_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.977
+// templ: version: v0.3.1001
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_ec_shards_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.977
+// templ: version: v0.3.1001
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_ec_volumes_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.977
+// templ: version: v0.3.1001
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_filers_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.977
+// templ: version: v0.3.1001
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.

weed/admin/view/app/cluster_masters_templ.go

@@ -1,6 +1,6 @@
 // Code generated by templ - DO NOT EDIT.
 
-// templ: version: v0.3.977
+// templ: version: v0.3.1001
 package app
 
 //lint:file-ignore SA4006 This context is only used if a nested component is present.