4.20

Update plugin_templ.go
build(deps): bump org.apache.kafka:kafka-clients from 3.9.1 to 3.9.2 in /test/kafka/kafka-client-loadtest/tools (#9056 )
2026-10-07 15:15:52 +00:00 · 2026-04-13 13:25:13 -07:00 · 2026-04-13 13:10:03 -07:00 · 2026-04-13 13:05:09 -07:00 · 2026-04-13 13:03:35 -07:00 · 2026-04-13 12:13:38 -07:00
321 changed files with 14200 additions and 3419 deletions
@@ -18,6 +18,7 @@ jobs:

      - name: Delete old release assets
        uses: mknejp/delete-release-assets@v1
+        continue-on-error: true
        with:
          token: ${{ github.token }}
          tag: dev
@@ -142,7 +142,7 @@ jobs:
          password: ${{ secrets.GHCR_TOKEN }}

      - name: Build
-        uses: docker/build-push-action@v7
+        uses: docker/build-push-action@v7.1.0
        with:
          context: ./docker
          push: ${{ github.event_name != 'pull_request' }}
@@ -150,7 +150,7 @@ jobs:
          fi

      - name: Build and push image
-        uses: docker/build-push-action@v7
+        uses: docker/build-push-action@v7.1.0
        with:
          context: ./docker
          push: ${{ github.event_name != 'pull_request' }}
@@ -232,7 +232,7 @@ jobs:
          username: ${{ secrets.GHCR_USERNAME }}
          password: ${{ secrets.GHCR_TOKEN }}
      - name: Build ${{ matrix.platform }} ${{ matrix.variant }}
-        uses: docker/build-push-action@v7
+        uses: docker/build-push-action@v7.1.0
        env:
          DOCKER_BUILDKIT: 1
        with:
@@ -322,7 +322,7 @@ jobs:
          buildkitd-config: /tmp/buildkitd.toml
      - name: Build local scan image tarball
        if: needs.setup.outputs.publish != 'true'
-        uses: docker/build-push-action@v7
+        uses: docker/build-push-action@v7.1.0
        env:
          DOCKER_BUILDKIT: 1
        with:
@@ -57,7 +57,7 @@ jobs:
          fi
      -
        name: Build
-        uses: docker/build-push-action@v7
+        uses: docker/build-push-action@v7.1.0
        with:
          context: ./docker
          push: ${{ github.event_name != 'pull_request' }}
@@ -251,7 +251,7 @@ jobs:
          
      - name: Build and push ${{ matrix.variant }}
        if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
-        uses: docker/build-push-action@v7
+        uses: docker/build-push-action@v7.1.0
        env:
          DOCKER_BUILDKIT: 1
        with:
@@ -94,7 +94,7 @@ jobs:
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Build and push image
-        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
+        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v2
        with:
          context: ./docker
          push: true
@@ -0,0 +1,63 @@
+name: "FUSE DLM Integration Tests"
+
+on:
+  pull_request:
+    paths:
+      - 'weed/command/mount*.go'
+      - 'weed/mount/**'
+      - 'weed/cluster/**'
+      - 'test/fuse_dlm/**'
+      - '.github/workflows/fuse-dlm-integration.yml'
+  push:
+    branches: [master]
+    paths:
+      - 'weed/command/mount*.go'
+      - 'weed/mount/**'
+      - 'weed/cluster/**'
+      - 'test/fuse_dlm/**'
+
+concurrency:
+  group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  fuse-dlm-integration:
+    name: FUSE DLM Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Install FUSE dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libfuse3-dev
+          echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
+          sudo chmod 644 /etc/fuse.conf
+
+      - name: Build SeaweedFS
+        run: go build -o weed/weed -buildvcs=false ./weed
+
+      - name: Run DLM integration tests
+        timeout-minutes: 25
+        env:
+          WEED_BINARY: ${{ github.workspace }}/weed/weed
+        run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
+
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: fuse-dlm-test-logs
+          path: /tmp/seaweedfs-fuse-dlm-logs/
+          retention-days: 3
@@ -0,0 +1,87 @@
+name: "pjdfstest POSIX Compliance"
+
+on:
+  push:
+    branches: [ master, main ]
+    paths:
+      - 'weed/mount/**'
+      - 'weed/filer/**'
+      - 'test/pjdfstest/**'
+      - '.github/workflows/pjdfstest.yml'
+  pull_request:
+    branches: [ master, main ]
+    paths:
+      - 'weed/mount/**'
+      - 'weed/filer/**'
+      - 'test/pjdfstest/**'
+      - '.github/workflows/pjdfstest.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: pjdfstest/${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  pjdfstest:
+    name: pjdfstest
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v6
+
+    - name: Set up Go
+      uses: actions/setup-go@v6
+      with:
+        go-version-file: 'go.mod'
+
+    - name: Build SeaweedFS e2e image
+      run: |
+        cd docker
+        make build_e2e || {
+          echo "Retrying without buildx cache..."
+          make binary_race
+          docker build --no-cache -t chrislusf/seaweedfs:e2e -f Dockerfile.e2e .
+        }
+
+    - name: Build pjdfstest image
+      run: |
+        docker build -t chrislusf/seaweedfs:pjdfstest test/pjdfstest/
+
+    - name: Start SeaweedFS cluster
+      run: |
+        docker compose -f test/pjdfstest/docker-compose.yml up --wait
+
+    - name: Run pjdfstest
+      run: |
+        set -o pipefail
+        docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
+          /run.sh 2>&1 | tee /tmp/pjdfstest-output.log
+
+    - name: Collect logs
+      if: always()
+      run: |
+        mkdir -p /tmp/pjdfstest-docker-logs
+        for svc in master volume filer mount; do
+          docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
+            > "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
+        done
+
+    - name: Tear down
+      if: always()
+      run: |
+        docker compose -f test/pjdfstest/docker-compose.yml down -v
+
+    - name: Upload logs
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: pjdfstest-results
+        path: |
+          /tmp/pjdfstest-output.log
+          /tmp/pjdfstest-docker-logs/
+        retention-days: 7
@@ -19,6 +19,7 @@ jobs:
    steps:
      - name: Delete old Rust volume dev assets
        uses: mknejp/delete-release-assets@v1
+        continue-on-error: true
        with:
          token: ${{ github.token }}
          tag: dev
@@ -83,7 +84,7 @@ jobs:
          rm weed-volume-normal

      - name: Upload dev release assets
-        uses: softprops/action-gh-release@v2
+        uses: softprops/action-gh-release@v3
        with:
          tag_name: dev
          prerelease: true
@@ -154,7 +155,7 @@ jobs:
          rm weed-volume-normal

      - name: Upload dev release assets
-        uses: softprops/action-gh-release@v2
+        uses: softprops/action-gh-release@v3
        with:
          tag_name: dev
          prerelease: true
@@ -89,7 +89,7 @@ jobs:

      - name: Upload release assets
        if: startsWith(github.ref, 'refs/tags/')
-        uses: softprops/action-gh-release@v2
+        uses: softprops/action-gh-release@v3
        with:
          files: |
            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
@@ -166,7 +166,7 @@ jobs:

      - name: Upload release assets
        if: startsWith(github.ref, 'refs/tags/')
-        uses: softprops/action-gh-release@v2
+        uses: softprops/action-gh-release@v3
        with:
          files: |
            weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
@@ -235,7 +235,7 @@ jobs:

      - name: Upload release assets
        if: startsWith(github.ref, 'refs/tags/')
-        uses: softprops/action-gh-release@v2
+        uses: softprops/action-gh-release@v3
        with:
          files: |
            weed-volume_large_disk_windows_amd64.zip
@@ -0,0 +1,56 @@
+name: "Vacuum Integration Tests"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+permissions:
+  contents: read
+
+jobs:
+  vacuum-integration-tests:
+    name: Vacuum Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 15
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v6
+      with:
+        go-version: ^1.25
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v6
+
+    - name: Build weed binary
+      run: |
+        cd weed && go build -o weed .
+
+    - name: Run Vacuum Integration Tests
+      working-directory: test/vacuum
+      run: |
+        go test -v -timeout 10m
+
+    - name: Collect server logs on failure
+      if: failure()
+      run: |
+        echo "Collecting server logs from temp directories..."
+        mkdir -p /tmp/vacuum-test-logs
+        find /tmp -maxdepth 1 -type d -name "TestVacuum*" 2>/dev/null | while read dir; do
+          if [ -d "$dir" ]; then
+            echo "Found test directory: $dir"
+            cp -r "$dir" /tmp/vacuum-test-logs/ 2>/dev/null || true
+          fi
+        done
+        echo "Collected logs:"
+        find /tmp/vacuum-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
+
+    - name: Archive logs
+      if: failure()
+      uses: actions/upload-artifact@v7
+      with:
+        name: vacuum-integration-test-logs
+        path: /tmp/vacuum-test-logs/
+        retention-days: 14
@@ -42,8 +42,7 @@ test: admin-generate

 # Admin component targets
 admin-generate:
-	@echo "Generating admin component templates..."
-	@cd $(ADMIN_DIR) && templ generate ./view
+	@cd $(ADMIN_DIR) && $(MAKE) generate

 admin-build: admin-generate
 	@echo "Building admin component..."
@@ -92,14 +92,14 @@ require (
 	gocloud.dev v0.45.0
 	gocloud.dev/pubsub/natspubsub v0.45.0
 	gocloud.dev/pubsub/rabbitpubsub v0.45.0
-	golang.org/x/crypto v0.49.0
+	golang.org/x/crypto v0.50.0
 	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
 	golang.org/x/image v0.38.0
-	golang.org/x/net v0.52.0
+	golang.org/x/net v0.53.0
 	golang.org/x/oauth2 v0.36.0
-	golang.org/x/sys v0.42.0
-	golang.org/x/text v0.35.0 // indirect
-	golang.org/x/tools v0.42.0 // indirect
+	golang.org/x/sys v0.43.0
+	golang.org/x/text v0.36.0 // indirect
+	golang.org/x/tools v0.43.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/api v0.274.0
 	google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 // indirect
@@ -118,15 +118,15 @@ require (
 	github.com/Jille/raft-grpc-transport v1.6.1
 	github.com/ThreeDotsLabs/watermill v1.5.1
 	github.com/a-h/templ v0.3.977
-	github.com/apache/cassandra-gocql-driver/v2 v2.0.0
+	github.com/apache/cassandra-gocql-driver/v2 v2.1.0
 	github.com/apache/iceberg-go v0.5.0
 	github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
 	github.com/arangodb/go-driver v1.6.9
 	github.com/armon/go-metrics v0.4.1
 	github.com/aws/aws-sdk-go-v2 v1.41.5
-	github.com/aws/aws-sdk-go-v2/config v1.32.13
+	github.com/aws/aws-sdk-go-v2/config v1.32.14
 	github.com/aws/aws-sdk-go-v2/credentials v1.19.14
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
 	github.com/getsentry/sentry-go v0.44.1
@@ -154,7 +154,7 @@ require (
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.134.0
 	go.etcd.io/etcd/client/pkg/v3 v3.6.10
 	go.uber.org/atomic v1.11.0
 	golang.org/x/sync v0.20.0
@@ -282,8 +282,8 @@ require (
 	go.uber.org/mock v0.5.2 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/mod v0.33.0 // indirect
-	golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect
+	golang.org/x/mod v0.34.0 // indirect
+	golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c // indirect
 	gonum.org/v1/gonum v0.17.0 // indirect
 )

@@ -476,7 +476,7 @@ require (
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
-	github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 // indirect
 	github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
 	github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
@@ -489,14 +489,14 @@ require (
 	go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
-	go.opentelemetry.io/otel v1.42.0 // indirect
-	go.opentelemetry.io/otel/metric v1.42.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.42.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.42.0 // indirect
-	go.opentelemetry.io/otel/trace v1.42.0 // indirect
+	go.opentelemetry.io/otel v1.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.43.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/trace v1.43.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.1 // indirect
-	golang.org/x/term v0.41.0 // indirect
+	golang.org/x/term v0.42.0
 	golang.org/x/time v0.15.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
@@ -692,8 +692,8 @@ github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmO
 github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 h1:r/gg2gzUXiXoy72VU3jnODh8l/5rL0aslnTAbtmai/U=
 github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87/go.mod h1:IJTMBTlHe7cDOhRh0ioGuEKBl5iTR6xPfl5BN4AgirU=
 github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
-github.com/apache/cassandra-gocql-driver/v2 v2.0.0 h1:Omnzb1Z/P90Dr2TbVNu54ICQL7TKVIIsJO231w484HU=
-github.com/apache/cassandra-gocql-driver/v2 v2.0.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
+github.com/apache/cassandra-gocql-driver/v2 v2.1.0 h1:VEbbeJ2ift4deKMZ6Fs55Vs3fq/RrkjCcxCnqUxhwf8=
+github.com/apache/cassandra-gocql-driver/v2 v2.1.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
 github.com/apache/iceberg-go v0.5.0 h1:wQj4CK5YiXZcB+tj19gWG+Jf1I6MiORQ/StSL/E5gGQ=
 github.com/apache/iceberg-go v0.5.0/go.mod h1:F/rdP1yZmnO4mQ0Qew2HTGdc+ZV57cRfxbbq/uJm1eM=
 github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
@@ -718,8 +718,8 @@ github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV
 github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
-github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
-github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
+github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI=
+github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo=
 github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
 github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
@@ -742,8 +742,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3x
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 h1:foqo/ocQ7WqKwy3FojGtZQJo0FR4vto9qnz9VaumbCo=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
 github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
@@ -2058,14 +2058,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD
 github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
-github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q=
-github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
+github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 h1:avIdi8eGXjKbn1WLokNR1Ofnz1k8t7tJ88YQLD/iCi8=
+github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 h1:XaRxeVrOyl3y6v9CiYMWaFdZ6zevvYe+TRxOR8ifa2s=
 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3 h1:hTwpF+PdbuR7vcixN+4AC6yu4asaUIAljQCxL51eDII=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3/go.mod h1:stS1mQYjbJvwwYaYzKyFY9eMiuVXWWXQA6T+SpOLg9c=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.134.0 h1:Voog4d56wPNT8JQvbgN6QPdZ1gKG36YM5615PJB/XpM=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.134.0/go.mod h1:VYUUkRJkKuQPkIpgtZJj6+58Fa2g8ccAqdmaaK6HP5k=
 github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
 github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
 github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
@@ -2128,8 +2128,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0/go.mod h1:CosX/aS4eHnG9D7nESYpV753l4j9q5j3SL/PUYd2lR8=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
-go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
-go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
+go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
+go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8=
@@ -2144,14 +2144,14 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
 go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
 go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
-go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
-go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
-go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
-go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
-go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
-go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
-go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
-go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
+go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
+go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
+go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
+go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
+go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
+go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
+go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
+go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
@@ -2208,8 +2208,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
-golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
+golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
+golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2275,8 +2275,8 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
-golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
+golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
+golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2344,8 +2344,8 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
-golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
+golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
+golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2503,11 +2503,11 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
-golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
+golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
-golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 h1:bTLqdHv7xrGlFbvf5/TXNxy/iUwwdkjhqQTJDjW7aj0=
-golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548=
+golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c h1:6a8FdnNk6bTXBjR4AGKFgUKuo+7GnR3FX5L7CbveeZc=
+golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2523,8 +2523,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
-golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
-golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
+golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
+golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2545,8 +2545,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
-golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
+golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
+golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2624,8 +2624,8 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
-golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
+golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
+golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
 golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
 golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.18"
+appVersion: "4.20"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.18.0
+version: 4.20.0
@@ -49,6 +49,35 @@ CREATE TABLE IF NOT EXISTS `filemeta` (

 Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.

+#### RocksDB variant
+
+The `_large_disk_rocksdb` image tag ships with RocksDB pre-configured as the filer backend.
+To use this image with the Helm chart, override the image on all three components and disable
+the chart's default `WEED_LEVELDB2_ENABLED`, which would otherwise re-enable LevelDB2 and
+override the image's built-in RocksDB configuration:
+
+```yaml
+# Replace <VERSION> with the desired seaweedfs version, e.g. 3.80_large_disk_rocksdb.
+master:
+  imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
+
+volume:
+  imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
+
+filer:
+  enablePVC: true
+  imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
+  extraEnvironmentVars:
+    WEED_LEVELDB2_ENABLED: "false"
+```
+
+Notes:
+
+* `master` and `volume` use the same image tag so that all components share a consistent
+  SeaweedFS build; RocksDB itself is only used by the filer.
+* `filer.enablePVC: true` (or another form of persistent storage for the filer) is required
+  so that the RocksDB metadata store survives pod restarts — otherwise metadata will be lost.
+
 ### Node Labels
 Kubernetes nodes can have labels which help to define which node(Host) will run which pod:

@@ -82,7 +82,7 @@ spec:
      {{- end }}
      containers:
        - name: seaweedfs
-          image: {{ template "admin.image" . }}
+          image: {{ template "seaweedfs.admin.image" . }}
          imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
          {{- $adminAuthEnabled := or .Values.admin.secret.existingSecret .Values.admin.secret.adminPassword }}
          {{- $urlPrefix := .Values.admin.urlPrefix }}
@@ -242,7 +242,7 @@ spec:
          securityContext: {{- omit .Values.admin.containerSecurityContext "enabled" | toYaml | nindent 12 }}
          {{- end }}
      {{- if .Values.admin.sidecars }}
-      {{- include "common.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
+      {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
      {{- end }}
      volumes:
        {{- if eq .Values.admin.data.type "hostPath" }}
@@ -303,7 +303,7 @@ spec:
      nodeSelector:
        {{ tpl .Values.admin.nodeSelector . | indent 8 | trim }}
      {{- end }}
-  {{- $pvc_exists := include "admin.pvc_exists" . -}}
+  {{- $pvc_exists := include "seaweedfs.admin.pvc_exists" . -}}
  {{- if $pvc_exists }}
  volumeClaimTemplates:
    {{- if eq .Values.admin.data.type "persistentVolumeClaim" }}
@@ -77,7 +77,7 @@ spec:
      {{- end }}
      containers:
        - name: seaweedfs
-          image: {{ template "master.image" . }}
+          image: {{ template "seaweedfs.master.image" . }}
          imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
          env:
            {{- /* Determine default cluster alias and the corresponding env var keys to avoid conflicts */}}
@@ -418,7 +418,7 @@ spec:
            {{- omit .Values.allInOne.containerSecurityContext "enabled" | toYaml | nindent 12 }}
          {{- end }}
      {{- if .Values.allInOne.sidecars }}
-      {{- include "common.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
+      {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
      {{- end }}
      volumes:
        - name: data
@@ -164,7 +164,7 @@ spec:
          securityContext: {{- omit .Values.cosi.containerSecurityContext "enabled" | toYaml | nindent 12 }}
          {{- end }}
      {{- if .Values.cosi.sidecars }}
-      {{- include "common.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
+      {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
      {{- end }}
      volumes:
        - name: socket
@@ -86,7 +86,7 @@ spec:
      {{- end }}
      containers:
        - name: seaweedfs
-          image: {{ template "filer.image" . }}
+          image: {{ template "seaweedfs.filer.image" . }}
          imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
          env:
            - name: POD_IP
@@ -317,7 +317,7 @@ spec:
          securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
          {{- end }}
        {{- if .Values.filer.sidecars }}
-        {{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
+        {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
        {{- end }}
      volumes:
        {{- if eq .Values.filer.logs.type "hostPath" }}
@@ -413,7 +413,7 @@ spec:
      storageClassName: {{ .Values.filer.storageClass }}
      {{- end }}
  {{- end }}
-  {{- $pvc_exists := include "filer.pvc_exists" . -}}
+  {{- $pvc_exists := include "seaweedfs.filer.pvc_exists" . -}}
  {{- if $pvc_exists }}
  volumeClaimTemplates:
    {{- if eq .Values.filer.data.type "persistentVolumeClaim" }}
@@ -80,7 +80,7 @@ spec:
      {{- end }}
      containers:
        - name: seaweedfs
-          image: {{ template "master.image" . }}
+          image: {{ template "seaweedfs.master.image" . }}
          imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
          env:
            - name: POD_IP
@@ -251,7 +251,7 @@ spec:
          securityContext: {{- omit .Values.master.containerSecurityContext "enabled" | toYaml | nindent 12 }}
          {{- end }}
        {{- if .Values.master.sidecars }}
-        {{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
+        {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
        {{- end }}
      volumes:
        {{- if eq .Values.master.logs.type "hostPath" }}
@@ -312,7 +312,7 @@ spec:
      nodeSelector:
        {{ tpl .Values.master.nodeSelector . | indent 8 | trim }}
      {{- end }}
-  {{- $pvc_exists := include "master.pvc_exists" . -}}
+  {{- $pvc_exists := include "seaweedfs.master.pvc_exists" . -}}
  {{- if $pvc_exists }}
  volumeClaimTemplates:
    {{- if eq .Values.master.data.type "persistentVolumeClaim"}}
@@ -74,7 +74,7 @@ spec:
      {{- end }}
      containers:
        - name: seaweedfs
-          image: {{ template "s3.image" . }}
+          image: {{ template "seaweedfs.s3.image" . }}
          imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
          env:
            - name: POD_IP
@@ -226,7 +226,7 @@ spec:
          securityContext: {{- omit .Values.s3.containerSecurityContext "enabled" | toYaml | nindent 12 }}
          {{- end }}
      {{- if .Values.s3.sidecars }}
-      {{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
+      {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
      {{- end }}
      volumes:
        {{- if .Values.s3.enableAuth }}
@@ -15,15 +15,15 @@
 {{- $access_key_admin := $adminCreds.accessKey -}}
 {{- $secret_key_admin := $adminCreds.secretKey -}}
 {{- if not (and $access_key_admin $secret_key_admin) -}}
-  {{- $access_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
-  {{- $secret_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
+  {{- $access_key_admin = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
+  {{- $secret_key_admin = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
 {{- end -}}
 {{- $readCreds := $creds.read | default dict -}}
 {{- $access_key_read := $readCreds.accessKey -}}
 {{- $secret_key_read := $readCreds.secretKey -}}
 {{- if not (and $access_key_read $secret_key_read) -}}
-  {{- $access_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
-  {{- $secret_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
+  {{- $access_key_read = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
+  {{- $secret_key_read = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
 {{- end -}}
 apiVersion: v1
 kind: Secret
@@ -74,7 +74,7 @@ spec:
      {{- end }}
      containers:
        - name: seaweedfs
-          image: {{ template "sftp.image" . }}
+          image: {{ template "seaweedfs.sftp.image" . }}
          imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
          env:
            - name: POD_IP
@@ -233,7 +233,7 @@ spec:
          securityContext: {{- omit .Values.sftp.containerSecurityContext "enabled" | toYaml | nindent 12 }}
          {{- end }}
      {{- if .Values.sftp.sidecars }}
-      {{- include "common.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
+      {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
      {{- end }}
      volumes:
        {{- if .Values.sftp.enableAuth }}
@@ -1,8 +1,8 @@
 {{- if or .Values.sftp.enabled .Values.allInOne.enabled }}
 {{- $secretName := printf "%s-sftp-secret" (include "seaweedfs.fullname" .) }}
-{{- $admin_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_password" 20) -}}
-{{- $read_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "readonly_password" 20) -}}
-{{- $public_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "public_user_password" 20) -}}
+{{- $admin_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_password" "length" 20) -}}
+{{- $read_user_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "readonly_password" "length" 20) -}}
+{{- $public_user_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "public_user_password" "length" 20) -}}
 apiVersion: v1
 kind: Secret
 type: Opaque
@@ -72,77 +72,77 @@ Inject extra environment vars in the format key:value, if populated
 {{- end -}}

 {{/* Return the proper filer image */}}
-{{- define "filer.image" -}}
+{{- define "seaweedfs.filer.image" -}}
 {{- if .Values.filer.imageOverride -}}
 {{- $imageOverride := .Values.filer.imageOverride -}}
 {{- printf "%s" $imageOverride -}}
 {{- else -}}
-{{- include "common.image" . }}
+{{- include "seaweedfs.image" . }}
 {{- end -}}
 {{- end -}}

 {{/* Return the proper master image */}}
-{{- define "master.image" -}}
+{{- define "seaweedfs.master.image" -}}
 {{- if .Values.master.imageOverride -}}
 {{- $imageOverride := .Values.master.imageOverride -}}
 {{- printf "%s" $imageOverride -}}
 {{- else -}}
-{{- include "common.image" . }}
+{{- include "seaweedfs.image" . }}
 {{- end -}}
 {{- end -}}

 {{/* Return the proper s3 image */}}
-{{- define "s3.image" -}}
+{{- define "seaweedfs.s3.image" -}}
 {{- if .Values.s3.imageOverride -}}
 {{- $imageOverride := .Values.s3.imageOverride -}}
 {{- printf "%s" $imageOverride -}}
 {{- else -}}
-{{- include "common.image" . }}
+{{- include "seaweedfs.image" . }}
 {{- end -}}
 {{- end -}}

 {{/* Return the proper sftp image */}}
-{{- define "sftp.image" -}}
+{{- define "seaweedfs.sftp.image" -}}
 {{- if .Values.sftp.imageOverride -}}
 {{- $imageOverride := .Values.sftp.imageOverride -}}
 {{- printf "%s" $imageOverride -}}
 {{- else -}}
-{{- include "common.image" . }}
+{{- include "seaweedfs.image" . }}
 {{- end -}}
 {{- end -}}

 {{/* Return the proper admin image */}}
-{{- define "admin.image" -}}
+{{- define "seaweedfs.admin.image" -}}
 {{- if .Values.admin.imageOverride -}}
 {{- $imageOverride := .Values.admin.imageOverride -}}
 {{- printf "%s" $imageOverride -}}
 {{- else -}}
-{{- include "common.image" . }}
+{{- include "seaweedfs.image" . }}
 {{- end -}}
 {{- end -}}

 {{/* Return the proper worker image */}}
-{{- define "worker.image" -}}
+{{- define "seaweedfs.worker.image" -}}
 {{- if .Values.worker.imageOverride -}}
 {{- $imageOverride := .Values.worker.imageOverride -}}
 {{- printf "%s" $imageOverride -}}
 {{- else -}}
-{{- include "common.image" . }}
+{{- include "seaweedfs.image" . }}
 {{- end -}}
 {{- end -}}

 {{/* Return the proper volume image */}}
-{{- define "volume.image" -}}
+{{- define "seaweedfs.volume.image" -}}
 {{- if .Values.volume.imageOverride -}}
 {{- $imageOverride := .Values.volume.imageOverride -}}
 {{- printf "%s" $imageOverride -}}
 {{- else -}}
-{{- include "common.image" . }}
+{{- include "seaweedfs.image" . }}
 {{- end -}}
 {{- end -}}

 {{/* Computes the container image name for all components (if they are not overridden) */}}
-{{- define "common.image" -}}
+{{- define "seaweedfs.image" -}}
 {{- $registryName := default .Values.image.registry .Values.global.imageRegistry | toString -}}
 {{- $repositoryName := default .Values.image.repository .Values.global.seaweedfs.image.repository | toString -}}
 {{- $name := .Values.global.seaweedfs.image.name | toString -}}
@@ -160,7 +160,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- end -}}

 {{/* check if any Volume PVC exists */}}
-{{- define "volume.pvc_exists" -}}
+{{- define "seaweedfs.volume.pvc_exists" -}}
 {{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
 {{- printf "true" -}}
 {{- else -}}
@@ -169,7 +169,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- end -}}

 {{/* check if any Filer PVC exists */}}
-{{- define "filer.pvc_exists" -}}
+{{- define "seaweedfs.filer.pvc_exists" -}}
 {{- if or (eq .Values.filer.data.type "persistentVolumeClaim") (eq .Values.filer.logs.type "persistentVolumeClaim") -}}
 {{- printf "true" -}}
 {{- else -}}
@@ -178,7 +178,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- end -}}

 {{/* check if any Master PVC exists */}}
-{{- define "master.pvc_exists" -}}
+{{- define "seaweedfs.master.pvc_exists" -}}
 {{- if or (eq .Values.master.data.type "persistentVolumeClaim") (eq .Values.master.logs.type "persistentVolumeClaim") -}}
 {{- printf "true" -}}
 {{- else -}}
@@ -187,7 +187,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- end -}}

 {{/* check if any Admin PVC exists */}}
-{{- define "admin.pvc_exists" -}}
+{{- define "seaweedfs.admin.pvc_exists" -}}
 {{- if or (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.logs.type "persistentVolumeClaim") -}}
 {{- printf "true" -}}
 {{- else -}}
@@ -196,7 +196,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- end -}}

 {{/* check if any InitContainers exist for Volumes */}}
-{{- define "volume.initContainers_exists" -}}
+{{- define "seaweedfs.volume.initContainers_exists" -}}
 {{- if or (not (empty .Values.volume.idx )) (not (empty .Values.volume.initContainers )) -}}
 {{- printf "true" -}}
 {{- else -}}
@@ -225,10 +225,10 @@ imagePullSecrets:
 {{/*
 Renders a value that contains template perhaps with scope if the scope is present.
 Usage:
-{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
-{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
+{{ include "seaweedfs.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
+{{ include "seaweedfs.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
 */}}
-{{- define "common.tplvalues.render" -}}
+{{- define "seaweedfs.tplvalues.render" -}}
 {{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }}
 {{- if contains "{{" (toJson .value) }}
  {{- if .scope }}
@@ -245,9 +245,9 @@ Usage:
 Converts a Kubernetes quantity like "256Mi" or "2G" to a float64 in base units,
 handling both binary (Ki, Mi, Gi) and decimal (m, k, M) suffixes; numeric inputs
 Usage:
-{{ include "common.resource-quantity" "10Gi" }}
+{{ include "seaweedfs.resource-quantity" "10Gi" }}
 */}}
-{{- define "common.resource-quantity" -}}
+{{- define "seaweedfs.resource-quantity" -}}
    {{- $value := . -}}
    {{- $unit := 1.0 -}}
    {{- if typeIs "string" . -}}
@@ -267,7 +267,7 @@ Usage:
 getOrGeneratePassword will check if a password exists in a secret and return it,
 or generate a new random password if it doesn't exist.
 */}}
-{{- define "getOrGeneratePassword" -}}
+{{- define "seaweedfs.getOrGeneratePassword" -}}
 {{- $params := . -}}
 {{- $namespace := $params.namespace -}}
 {{- $secretName := $params.secretName -}}
@@ -68,7 +68,7 @@ spec:
      {{- include "seaweedfs.imagePullSecrets" $ | nindent 6 }}
      containers:
      - name: post-install-job
-        image: {{ template "master.image" . }}
+        image: {{ template "seaweedfs.master.image" . }}
        imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
        env:
          - name: WEED_CLUSTER_DEFAULT
@@ -35,8 +35,8 @@
 {{-             $pvcName := printf "%s-%s-%s-%d" $dir.name $seaweedfsName $volumeName $e }}
 {{-             $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) }}
 {{-             if and $currentPVC }}
-{{-             $oldSize := include "common.resource-quantity" $currentPVC.spec.resources.requests.storage }}
-{{-             $newSize := include "common.resource-quantity" $desiredSize }}
+{{-             $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage }}
+{{-             $newSize := include "seaweedfs.resource-quantity" $desiredSize }}
 {{-               if gt $newSize $oldSize }}
 {{-               $commands = append $commands (printf "kubectl patch pvc %s-%s-%s-%d -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"%s\"}}}}'" $dir.name $seaweedfsName $volumeName $e $desiredSize) }}
 {{-               end }}
@@ -71,12 +71,12 @@ spec:
      {{- end }}
      enableServiceLinks: false
      serviceAccountName: {{ $volume.serviceAccountName | default (include "seaweedfs.serviceAccountName" $) | quote }} # for deleting statefulset pods after migration
-      {{- $initContainers_exists := include "volume.initContainers_exists" $ -}}
+      {{- $initContainers_exists := include "seaweedfs.volume.initContainers_exists" $ -}}
      {{- if $initContainers_exists }}
      initContainers:
        {{- if $volume.idx }}
        - name: seaweedfs-vol-move-idx
-          image: {{ template "volume.image" $ }}
+          image: {{ template "seaweedfs.volume.image" $ }}
          imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
          command: [ '/bin/sh', '-c' ]
          args: [ '{{range $dir :=  $volume.dataDirs }}if ls /{{$dir.name}}/*.idx  >/dev/null 2>&1; then mv /{{$dir.name}}/*.idx /idx/ ; fi; {{end}}' ]
@@ -104,7 +104,7 @@ spec:
      {{- end }}
      containers:
        - name: seaweedfs
-          image: {{ template "volume.image" $ }}
+          image: {{ template "seaweedfs.volume.image" $ }}
          imagePullPolicy: {{ default "IfNotPresent" $.Values.global.seaweedfs.imagePullPolicy }}
          env:
            - name: POD_NAME
@@ -274,7 +274,7 @@ spec:
          securityContext: {{- omit $volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}
          {{- end }}
      {{- if $volume.sidecars }}
-      {{- include "common.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
+      {{- include "seaweedfs.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
      {{- end }}
      volumes:

@@ -77,7 +77,7 @@ spec:
      {{- end }}
      containers:
        - name: seaweedfs
-          image: {{ template "worker.image" . }}
+          image: {{ template "seaweedfs.worker.image" . }}
          imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
          env:
            - name: POD_IP
@@ -219,7 +219,7 @@ spec:
          securityContext: {{- omit .Values.worker.containerSecurityContext "enabled" | toYaml | nindent 12 }}
          {{- end }}
      {{- if .Values.worker.sidecars }}
-      {{- include "common.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
+      {{- include "seaweedfs.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
      {{- end }}
      volumes:
        {{- if eq .Values.worker.data.type "hostPath" }}
@@ -1339,9 +1339,14 @@ worker:
  # Admin server to connect to
  adminServer: ""

-  # Worker job types - comma-separated list
-  # Available: vacuum, volume_balance, erasure_coding
-  jobType: "vacuum,volume_balance,erasure_coding"
+  # Worker job types - categories or comma-separated list of names/aliases
+  # Categories: all, default, heavy
+  #   default: vacuum, volume_balance, ec_balance, admin_script
+  #   heavy: erasure_coding, iceberg_maintenance
+  #   all: every registered job type
+  # Examples: "all", "default", "heavy", "default,iceberg" (default+iceberg), "vacuum,volume_balance"
+  # Refer: https://github.com/seaweedfs/seaweedfs/wiki/Worker
+  jobType: "all"

  # Maximum number of concurrent detection requests
  maxDetect: 1
@@ -294,6 +294,7 @@ message AssignVolumeRequest {
    string rack = 7;
    string data_node = 9;
    string disk_type = 8;
+    uint64 expected_data_size = 10; // hint for size-aware volume selection
 }

 message AssignVolumeResponse {
@@ -231,6 +231,7 @@ message AssignRequest {
  uint32 memory_map_max_size_mb = 8;
  uint32 writable_volume_count = 9;
  string disk_type = 10;
+  uint64 expected_data_size = 11; // hint for size-aware volume selection
 }

 message VolumeGrowRequest {
@@ -486,6 +486,11 @@ impl S3TierRegistry {
        self.backends.keys().cloned().collect()
    }

+    /// Remove a backend by name.
+    pub fn remove(&mut self, name: &str) {
+        self.backends.remove(name);
+    }
+
    pub fn clear(&mut self) {
        self.backends.clear();
    }
@@ -4192,6 +4192,7 @@ mod tests {
        let addr = listener.local_addr().unwrap();
        listener.set_nonblocking(true).unwrap();
        let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
+        let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();

        std::thread::spawn(move || {
            let runtime = tokio::runtime::Builder::new_current_thread()
@@ -4253,6 +4254,7 @@ mod tests {
                }));

                let listener = tokio::net::TcpListener::from_std(listener).unwrap();
+                let _ = ready_tx.send(());
                axum::serve(listener, app)
                    .with_graceful_shutdown(async move {
                        let _ = shutdown_rx.await;
@@ -4262,6 +4264,8 @@ mod tests {
            });
        });

+        // Wait for the server thread to be ready before returning.
+        ready_rx.recv().unwrap();
        (format!("http://{}", addr), shutdown_tx)
    }

@@ -4307,7 +4311,9 @@ mod tests {
        std::fs::remove_file(&dat_path).unwrap();

        let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
-        global_s3_tier_registry().write().unwrap().clear();
+        // Use a test-specific backend_id to avoid racing with other tests
+        // that share the global registry.  Never call clear() — only
+        // register/remove our own entries.
        let tier_config = S3TierConfig {
            access_key: "access".to_string(),
            secret_key: "secret".to_string(),
@@ -4319,14 +4325,16 @@ mod tests {
        };
        {
            let mut registry = global_s3_tier_registry().write().unwrap();
-            registry.register("s3.default".to_string(), S3TierBackend::new(&tier_config));
-            registry.register("s3".to_string(), S3TierBackend::new(&tier_config));
+            registry.register(
+                "s3.incr_copy_test".to_string(),
+                S3TierBackend::new(&tier_config),
+            );
        }

        let vif = crate::storage::volume::VifVolumeInfo {
            files: vec![crate::storage::volume::VifRemoteFile {
                backend_type: "s3".to_string(),
-                backend_id: "default".to_string(),
+                backend_id: "incr_copy_test".to_string(),
                key: "remote-key".to_string(),
                offset: 0,
                file_size: dat_bytes.len() as u64,
@@ -4535,7 +4543,10 @@ mod tests {
        assert_eq!(copied, dat_bytes[super_block_size as usize..]);

        let _ = shutdown_tx.send(());
-        global_s3_tier_registry().write().unwrap().clear();
+        global_s3_tier_registry()
+            .write()
+            .unwrap()
+            .remove("s3.incr_copy_test");
    }

    #[tokio::test]
@@ -1408,10 +1408,8 @@ mod tests {
    #[test]
    fn test_apply_storage_backends_registers_s3_default_aliases() {
        let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
-        crate::remote_storage::s3_tier::global_s3_tier_registry()
-            .write()
-            .unwrap()
-            .clear();
+        // Do not call clear() on the global registry — other tests may be
+        // running concurrently.  Just register our entries and verify them.

        apply_storage_backends(
            &state,
@@ -1443,10 +1441,8 @@ mod tests {
    #[test]
    fn test_apply_storage_backends_ignores_unsupported_types() {
        let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
-        crate::remote_storage::s3_tier::global_s3_tier_registry()
-            .write()
-            .unwrap()
-            .clear();
+        // Do not call clear() on the global registry — other tests may be
+        // running concurrently.

        apply_storage_backends(
            &state,
@@ -1457,12 +1453,17 @@ mod tests {
            }],
        );

+        // The per-state registry is freshly created and should have no entries
+        // since "rclone" is unsupported.
        let registry = state.s3_tier_registry.read().unwrap();
        assert!(registry.names().is_empty());
+        // Only check that the unsupported type was not added to the global
+        // registry.  Other tests may have their own entries present.
        let global_registry = crate::remote_storage::s3_tier::global_s3_tier_registry()
            .read()
            .unwrap();
-        assert!(global_registry.names().is_empty());
+        assert!(global_registry.get("rclone.default").is_none());
+        assert!(global_registry.get("rclone").is_none());
    }

    #[test]
@@ -1158,7 +1158,13 @@ mod tests {
                Vec::new(),
            )
            .unwrap();
-        store.volume_size_limit.store(1024, Ordering::Relaxed);
+        // Use a large volume_size_limit so the unused-space difference between
+        // preallocate=true (0) and preallocate=false (~2 × limit) is big enough
+        // that integer-division rounding and disk-free fluctuations between the
+        // two get_disk_stats calls cannot make the quotients equal.
+        store
+            .volume_size_limit
+            .store(100 * 1024 * 1024, Ordering::Relaxed);
        store
            .add_volume(
                VolumeId(61),
@@ -3225,6 +3225,7 @@ mod tests {
        let addr = listener.local_addr().unwrap();
        listener.set_nonblocking(true).unwrap();
        let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
+        let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();

        std::thread::spawn(move || {
            let runtime = tokio::runtime::Builder::new_current_thread()
@@ -3279,6 +3280,7 @@ mod tests {
                }));

                let listener = tokio::net::TcpListener::from_std(listener).unwrap();
+                let _ = ready_tx.send(());
                axum::serve(listener, app)
                    .with_graceful_shutdown(async move {
                        let _ = shutdown_rx.await;
@@ -3288,6 +3290,8 @@ mod tests {
            });
        });

+        // Wait for the server thread to be ready before returning.
+        ready_rx.recv().unwrap();
        (format!("http://{}", addr), shutdown_tx)
    }

@@ -3833,7 +3837,7 @@ mod tests {
            let vif = VifVolumeInfo {
                files: vec![VifRemoteFile {
                    backend_type: "s3".to_string(),
-                    backend_id: "default".to_string(),
+                    backend_id: "vif_rw_test".to_string(),
                    key: "remote-key".to_string(),
                    offset: 0,
                    file_size: v.dat_file_size().unwrap(),
@@ -3849,6 +3853,27 @@ mod tests {
            )
            .unwrap();

+            // Register a dummy S3 backend so load_remote_dat_file can look it
+            // up.  This test never reads remote data, so a dead endpoint is fine.
+            // Use a test-specific backend_id to avoid racing with other tests
+            // that share the global registry.
+            let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
+                access_key: "access".to_string(),
+                secret_key: "secret".to_string(),
+                region: "us-east-1".to_string(),
+                bucket: "bucket-a".to_string(),
+                endpoint: "http://127.0.0.1:1".to_string(),
+                storage_class: "STANDARD".to_string(),
+                force_path_style: true,
+            };
+            crate::remote_storage::s3_tier::global_s3_tier_registry()
+                .write()
+                .unwrap()
+                .register(
+                    "s3.vif_rw_test".to_string(),
+                    crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
+                );
+
            v.dat_file_size().unwrap()
        };

@@ -3892,6 +3917,11 @@ mod tests {
            .unwrap();
        assert!(deleted_size.0 > 0);
        assert_eq!(v.dat_file_size().unwrap(), dat_size_before_reload);
+
+        crate::remote_storage::s3_tier::global_s3_tier_registry()
+            .write()
+            .unwrap()
+            .remove("s3.vif_rw_test");
    }

    #[test]
@@ -4054,10 +4084,9 @@ mod tests {
        std::fs::remove_file(&dat_path).unwrap();

        let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
-        crate::remote_storage::s3_tier::global_s3_tier_registry()
-            .write()
-            .unwrap()
-            .clear();
+        // Use a test-specific backend_id to avoid racing with other tests
+        // that share the global registry.  Never call clear() — only
+        // register/remove our own entries.
        let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
            access_key: "access".to_string(),
            secret_key: "secret".to_string(),
@@ -4072,11 +4101,7 @@ mod tests {
                .write()
                .unwrap();
            registry.register(
-                "s3.default".to_string(),
-                crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
-            );
-            registry.register(
-                "s3".to_string(),
+                "s3.remote_only_rw".to_string(),
                crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
            );
        }
@@ -4084,7 +4109,7 @@ mod tests {
        let vif = VifVolumeInfo {
            files: vec![VifRemoteFile {
                backend_type: "s3".to_string(),
-                backend_id: "default".to_string(),
+                backend_id: "remote_only_rw".to_string(),
                key: "remote-key".to_string(),
                offset: 0,
                file_size: dat_bytes.len() as u64,
@@ -4141,6 +4166,10 @@ mod tests {
        assert_eq!(meta.data_size, 11);

        let _ = shutdown_tx.send(());
+        crate::remote_storage::s3_tier::global_s3_tier_registry()
+            .write()
+            .unwrap()
+            .remove("s3.remote_only_rw");
    }

    /// Volume destroy removes .vif alongside the primary data files.
@@ -33,11 +33,11 @@ require (
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/image v0.36.0 // indirect
-	golang.org/x/net v0.50.0 // indirect
+	golang.org/x/image v0.38.0 // indirect
+	golang.org/x/net v0.52.0 // indirect
 	golang.org/x/sys v0.42.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
+	golang.org/x/text v0.35.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
 	google.golang.org/grpc v1.79.3 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 )
@@ -27,8 +27,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -84,35 +84,35 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
-go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
-go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
-go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
-go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
-go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
-go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
-go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
-go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
-go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
+go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
+go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
+go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
+go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
+go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
+go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
+go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
+go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
+go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
+go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
 go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
-golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
-golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
-golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
+golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
+golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
 golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
 gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
 google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
@@ -0,0 +1,277 @@
+package fuse_dlm
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestDLMConcurrentWritersSameFile verifies that two mounts writing to the same
+// file concurrently produce valid (non-corrupted) data. With DLM enabled, the
+// writes are serialized — one blocks until the other completes.
+//
+// Note: cross-mount read consistency depends on FUSE kernel cache invalidation
+// and filer metadata subscription, which are asynchronous. This test verifies
+// write integrity, not instant read convergence.
+func TestDLMConcurrentWritersSameFile(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	const iterations = 5
+	for iter := 0; iter < iterations; iter++ {
+		fileName := fmt.Sprintf("concurrent_write_%d.txt", iter)
+		payloadA := []byte(fmt.Sprintf("mount0-iteration-%d-payload-AAAA", iter))
+		payloadB := []byte(fmt.Sprintf("mount1-iteration-%d-payload-BBBB", iter))
+
+		var wg sync.WaitGroup
+		wg.Add(2)
+
+		go func() {
+			defer wg.Done()
+			err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), payloadA, 0644)
+			assert.NoError(t, err, "mount0 write iteration %d", iter)
+		}()
+
+		go func() {
+			defer wg.Done()
+			err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), payloadB, 0644)
+			assert.NoError(t, err, "mount1 write iteration %d", iter)
+		}()
+
+		wg.Wait()
+
+		// Verify file is readable and contains one of the expected payloads
+		// (read from mount0 — its own view is authoritative for write success).
+		content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
+		require.NoError(t, err, "read from mount0 iteration %d", iter)
+		validPayload := string(content) == string(payloadA) || string(content) == string(payloadB)
+		assert.True(t, validPayload,
+			"iteration %d: content must be one of the expected payloads, got: %q", iter, content)
+	}
+}
+
+// TestDLMRepeatedOpenWriteClose verifies that repeated open/write/close cycles
+// from both mounts all succeed without errors.
+func TestDLMRepeatedOpenWriteClose(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	const cycles = 20
+	fileName := "repeated_write.txt"
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		for i := 0; i < cycles; i++ {
+			data := []byte(fmt.Sprintf("mount0-cycle-%d", i))
+			err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), data, 0644)
+			assert.NoError(t, err, "mount0 cycle %d", i)
+		}
+	}()
+
+	go func() {
+		defer wg.Done()
+		for i := 0; i < cycles; i++ {
+			data := []byte(fmt.Sprintf("mount1-cycle-%d", i))
+			err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), data, 0644)
+			assert.NoError(t, err, "mount1 cycle %d", i)
+		}
+	}()
+
+	wg.Wait()
+
+	// File must be readable from at least one mount
+	content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
+	require.NoError(t, err)
+	assert.NotEmpty(t, content, "file must not be empty")
+}
+
+// TestDLMWriteBlocksSecondWriter verifies the core DLM guarantee: while one
+// mount has a file open for writing, another mount's write-open blocks until
+// the first mount closes the file.
+func TestDLMWriteBlocksSecondWriter(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	fileName := "blocking_test.txt"
+	path0 := filepath.Join(cluster.mountPoints[0], fileName)
+	path1 := filepath.Join(cluster.mountPoints[1], fileName)
+
+	// Mount 0 opens the file for writing and holds it open
+	f, err := os.OpenFile(path0, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	require.NoError(t, err, "mount0 open")
+	_, err = f.Write([]byte("mount0-holds-lock"))
+	require.NoError(t, err, "mount0 write")
+
+	// Mount 1 tries to write — should block (we use a goroutine with atomic flag)
+	var mount1Completed atomic.Bool
+	mount1Done := make(chan error, 1)
+	go func() {
+		err := os.WriteFile(path1, []byte("mount1-waited"), 0644)
+		mount1Completed.Store(true)
+		mount1Done <- err
+	}()
+
+	// Give mount 1 a moment — it should NOT complete while mount 0 holds the file open
+	time.Sleep(3 * time.Second)
+	require.False(t, mount1Completed.Load(),
+		"mount1 write must not complete while mount0 holds the file open")
+	t.Log("mount1 write is blocked as expected while mount0 holds the file")
+
+	// Mount 0 closes the file — this releases the DLM lock
+	require.NoError(t, f.Close(), "mount0 close")
+
+	// Mount 1 should now complete
+	select {
+	case err := <-mount1Done:
+		assert.NoError(t, err, "mount1 write after mount0 close")
+	case <-time.After(30 * time.Second):
+		t.Fatal("mount1 write did not complete within 30s after mount0 closed")
+	}
+}
+
+// TestDLMRenameWhileWriteOpen verifies that a rename is coordinated with DLM:
+// while mount0 has a file open for writing (re-opened after creation),
+// mount1 cannot rename it until mount0 closes the file.
+func TestDLMRenameWhileWriteOpen(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	origName := "rename_source.txt"
+	newName := "rename_dest.txt"
+
+	// Create and close the file first so it's flushed to the filer and
+	// visible on both mounts.
+	require.NoError(t, os.WriteFile(
+		filepath.Join(cluster.mountPoints[0], origName),
+		[]byte("initial-content"), 0644))
+	time.Sleep(2 * time.Second) // metadata propagation
+
+	// Verify mount1 can see the file
+	_, err := os.Stat(filepath.Join(cluster.mountPoints[1], origName))
+	require.NoError(t, err, "mount1 should see the file")
+
+	// Mount 0 re-opens the file for writing and holds it open
+	f, err := os.OpenFile(
+		filepath.Join(cluster.mountPoints[0], origName),
+		os.O_WRONLY|os.O_TRUNC, 0644)
+	require.NoError(t, err, "mount0 reopen")
+	_, err = f.Write([]byte("data-while-holding-lock"))
+	require.NoError(t, err, "mount0 write")
+
+	// Mount 1 tries to rename — should block because mount0 holds the
+	// DLM lock on the old path
+	var renameCompleted atomic.Bool
+	renameDone := make(chan error, 1)
+	go func() {
+		err := os.Rename(
+			filepath.Join(cluster.mountPoints[1], origName),
+			filepath.Join(cluster.mountPoints[1], newName))
+		renameCompleted.Store(true)
+		renameDone <- err
+	}()
+
+	// Rename must NOT complete while mount0 holds the file open
+	time.Sleep(3 * time.Second)
+	require.False(t, renameCompleted.Load(),
+		"rename must not complete while mount0 holds the file open")
+	t.Log("rename is blocked as expected while mount0 holds the file")
+
+	// Mount 0 closes → releases DLM lock → rename should proceed
+	require.NoError(t, f.Close(), "mount0 close")
+
+	select {
+	case err := <-renameDone:
+		assert.NoError(t, err, "rename after mount0 close")
+	case <-time.After(30 * time.Second):
+		t.Fatal("rename did not complete within 30s after mount0 closed")
+	}
+}
+
+// Note: Same-mount rename while a file is open for writing is not tested here
+// because macOS FUSE serializes operations on the same inode, causing a
+// kernel-level deadlock between the Rename handler's internal flush and the
+// pending Close. Same-mount coordination is already handled by the per-mount
+// fhLockTable and FUSE kernel serialization, so DLM is not needed for it.
+
+// TestDLMConcurrentRenames verifies that two concurrent renames of the same
+// file from different mounts don't corrupt metadata. DLM locks on both old
+// and new paths ensure renames are serialized.
+func TestDLMConcurrentRenames(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping DLM integration test in short mode")
+	}
+
+	cluster := startDLMTestCluster(t)
+	t.Cleanup(cluster.Stop)
+
+	// Create a file first
+	origPath := filepath.Join(cluster.mountPoints[0], "rename_race.txt")
+	require.NoError(t, os.WriteFile(origPath, []byte("original-content"), 0644))
+	time.Sleep(1 * time.Second) // propagation
+
+	// Both mounts try to rename the same file concurrently
+	var wg sync.WaitGroup
+	var errA, errB error
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		errA = os.Rename(
+			filepath.Join(cluster.mountPoints[0], "rename_race.txt"),
+			filepath.Join(cluster.mountPoints[0], "renamed_by_mount0.txt"),
+		)
+	}()
+
+	go func() {
+		defer wg.Done()
+		errB = os.Rename(
+			filepath.Join(cluster.mountPoints[1], "rename_race.txt"),
+			filepath.Join(cluster.mountPoints[1], "renamed_by_mount1.txt"),
+		)
+	}()
+
+	wg.Wait()
+
+	// At least one rename should succeed; the other may fail with ENOENT
+	// since the source was already moved.
+	succeeded := 0
+	if errA == nil {
+		succeeded++
+		t.Logf("mount0 rename succeeded")
+	} else {
+		t.Logf("mount0 rename failed: %v", errA)
+	}
+	if errB == nil {
+		succeeded++
+		t.Logf("mount1 rename succeeded")
+	} else {
+		t.Logf("mount1 rename failed: %v", errB)
+	}
+	assert.GreaterOrEqual(t, succeeded, 1, "at least one rename must succeed")
+}
@@ -0,0 +1,516 @@
+package fuse_dlm
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+const filerGroup = "fuse-dlm-test"
+
+// dlmTestCluster manages a full SeaweedFS cluster with 2 filers and 2 FUSE
+// mounts for testing DLM-based cross-mount write coordination.
+type dlmTestCluster struct {
+	t          testing.TB
+	baseDir    string
+	weedBinary string
+
+	masterPort     int
+	masterGrpcPort int
+	volumePort     int
+	volumeGrpcPort int
+	filerPorts     [2]int
+	filerGrpcPorts [2]int
+	mountPoints    [2]string
+
+	masterCmd *exec.Cmd
+	volumeCmd *exec.Cmd
+	filerCmds [2]*exec.Cmd
+	mountCmds [2]*exec.Cmd
+	logFiles  []*os.File
+
+	cleanupOnce sync.Once
+}
+
+func startDLMTestCluster(t testing.TB) *dlmTestCluster {
+	binary := findWeedBinary()
+	if binary == "" {
+		t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
+	}
+
+	baseDir, err := os.MkdirTemp("", "seaweedfs_fuse_dlm_test_")
+	require.NoError(t, err)
+
+	c := &dlmTestCluster{
+		t:          t,
+		baseDir:    baseDir,
+		weedBinary: binary,
+	}
+	// Register cleanup early so processes are stopped even if a require fails below.
+	t.Cleanup(c.Stop)
+
+	// Allocate ports: master(2) + volume(2) + filer0(2) + filer1(2) = 8
+	ports := allocatePorts(t, 8)
+	c.masterPort = ports[0]
+	c.masterGrpcPort = ports[1]
+	c.volumePort = ports[2]
+	c.volumeGrpcPort = ports[3]
+	c.filerPorts[0] = ports[4]
+	c.filerGrpcPorts[0] = ports[5]
+	c.filerPorts[1] = ports[6]
+	c.filerGrpcPorts[1] = ports[7]
+
+	// Write empty security.toml
+	configDir := filepath.Join(baseDir, "config")
+	require.NoError(t, os.MkdirAll(configDir, 0755))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(""), 0644))
+
+	// Start master
+	require.NoError(t, c.startMaster(configDir))
+	require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.masterPort), 30*time.Second),
+		"master not ready\n%s", c.tailLog("master"))
+
+	// Start volume
+	require.NoError(t, c.startVolume(configDir))
+	require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.volumePort), 30*time.Second),
+		"volume not ready\n%s", c.tailLog("volume"))
+
+	// Start 2 filers
+	for i := 0; i < 2; i++ {
+		require.NoError(t, c.startFiler(i, configDir))
+		require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[i]), 30*time.Second),
+			"filer %d not ready\n%s", i, c.tailLog(fmt.Sprintf("filer%d", i)))
+	}
+	require.NoError(t, c.waitForFilerCount(2, 30*time.Second), "filer group registration")
+	require.NoError(t, c.waitForLockRingConverged(30*time.Second), "lock ring convergence")
+
+	// Start 2 mounts, both pointing at filer0 for metadata consistency.
+	// (filer1 exists for the DLM lock ring but both mounts share filer0's
+	// metadata store since leveldb is per-filer.)
+	for i := 0; i < 2; i++ {
+		mp := filepath.Join(baseDir, fmt.Sprintf("mount%d", i))
+		require.NoError(t, os.MkdirAll(mp, 0755))
+		c.mountPoints[i] = mp
+		require.NoError(t, c.startMount(i, configDir))
+		require.NoError(t, c.waitForMount(mp, 30*time.Second),
+			"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
+	}
+
+	return c
+}
+
+func (c *dlmTestCluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		// Stop mounts first (triggers flush + DLM unlock).
+		// Use stopCmd for bounded wait to avoid hanging on wedged FUSE processes.
+		for i := 1; i >= 0; i-- {
+			stopCmd(c.mountCmds[i])
+			// Backup unmount in case FUSE teardown didn't clean up
+			exec.Command("fusermount3", "-u", c.mountPoints[i]).Run()
+			exec.Command("fusermount", "-u", c.mountPoints[i]).Run()
+		}
+		// Stop filers, volume, master
+		for i := 1; i >= 0; i-- {
+			stopCmd(c.filerCmds[i])
+		}
+		stopCmd(c.volumeCmd)
+		stopCmd(c.masterCmd)
+
+		for _, f := range c.logFiles {
+			f.Close()
+		}
+
+		// Copy logs for CI
+		c.copyLogsForCI()
+
+		if !c.t.Failed() {
+			os.RemoveAll(c.baseDir)
+		}
+
+		// Wait for ports to be fully released before the next test
+		// allocates new ports (avoids TIME_WAIT collisions).
+		time.Sleep(2 * time.Second)
+	})
+}
+
+// masterAddress returns the master address in the format that encodes both
+// HTTP and gRPC ports: "host:httpPort.grpcPort". This is the format that
+// SeaweedFS uses to communicate non-default gRPC ports between components.
+func (c *dlmTestCluster) masterAddress() string {
+	return string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
+}
+
+func (c *dlmTestCluster) startMaster(configDir string) error {
+	c.masterCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"master",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.masterPort),
+		"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
+		"-mdir="+filepath.Join(c.baseDir, "master"),
+	)
+	return c.startCmd(c.masterCmd, "master")
+}
+
+func (c *dlmTestCluster) startVolume(configDir string) error {
+	volDir := filepath.Join(c.baseDir, "volume")
+	if err := os.MkdirAll(volDir, 0755); err != nil {
+		return fmt.Errorf("create volume dir: %w", err)
+	}
+	c.volumeCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"volume",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.volumePort),
+		"-port.grpc="+strconv.Itoa(c.volumeGrpcPort),
+		"-master="+c.masterAddress(),
+		"-dir="+volDir,
+		"-max=10",
+	)
+	return c.startCmd(c.volumeCmd, "volume")
+}
+
+func (c *dlmTestCluster) startFiler(idx int, configDir string) error {
+	filerDir := filepath.Join(c.baseDir, fmt.Sprintf("filer%d", idx))
+	if err := os.MkdirAll(filerDir, 0755); err != nil {
+		return fmt.Errorf("create filer dir: %w", err)
+	}
+	c.filerCmds[idx] = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"filer",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.filerPorts[idx]),
+		"-port.grpc="+strconv.Itoa(c.filerGrpcPorts[idx]),
+		"-master="+c.masterAddress(),
+		"-filerGroup="+filerGroup,
+		"-defaultStoreDir="+filerDir,
+	)
+	return c.startCmd(c.filerCmds[idx], fmt.Sprintf("filer%d", idx))
+}
+
+func (c *dlmTestCluster) filerAddress(idx int) string {
+	return string(pb.NewServerAddress("127.0.0.1", c.filerPorts[idx], c.filerGrpcPorts[idx]))
+}
+
+func (c *dlmTestCluster) startMount(idx int, configDir string) error {
+	cacheDir := filepath.Join(c.baseDir, fmt.Sprintf("cache%d", idx))
+	if err := os.MkdirAll(cacheDir, 0755); err != nil {
+		return fmt.Errorf("create cache dir: %w", err)
+	}
+	c.mountCmds[idx] = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"mount",
+		"-filer="+c.filerAddress(0), // both mounts use filer0 for shared metadata
+		"-dir="+c.mountPoints[idx],
+		"-filer.path=/",
+		"-dirAutoCreate",
+		"-allowOthers=false",
+		"-cacheDir="+cacheDir,
+		"-dlm",
+	)
+	return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
+}
+
+func (c *dlmTestCluster) startCmd(cmd *exec.Cmd, name string) error {
+	logPath := filepath.Join(c.baseDir, "logs")
+	if err := os.MkdirAll(logPath, 0755); err != nil {
+		return fmt.Errorf("create log dir: %w", err)
+	}
+	logFile, err := os.Create(filepath.Join(logPath, name+".log"))
+	if err != nil {
+		return err
+	}
+	c.logFiles = append(c.logFiles, logFile)
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	return cmd.Start()
+}
+
+func (c *dlmTestCluster) tailLog(name string) string {
+	data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
+	if err != nil {
+		return fmt.Sprintf("(log %s not available: %v)", name, err)
+	}
+	const maxTail = 8192
+	if len(data) > maxTail {
+		data = data[len(data)-maxTail:]
+	}
+	return string(data)
+}
+
+func (c *dlmTestCluster) copyLogsForCI() {
+	ciLogDir := "/tmp/seaweedfs-fuse-dlm-logs"
+	os.MkdirAll(ciLogDir, 0755)
+	logsDir := filepath.Join(c.baseDir, "logs")
+	entries, err := os.ReadDir(logsDir)
+	if err != nil {
+		return
+	}
+	for _, e := range entries {
+		data, err := os.ReadFile(filepath.Join(logsDir, e.Name()))
+		if err != nil {
+			continue
+		}
+		os.WriteFile(filepath.Join(ciLogDir, e.Name()), data, 0644)
+	}
+}
+
+func (c *dlmTestCluster) waitForTCP(addr string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", addr, time.Second)
+		if err == nil {
+			conn.Close()
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("service at %s not ready within timeout", addr)
+}
+
+// waitForMount waits for a FUSE filesystem to actually be mounted at
+// mountPoint by comparing the device ID of the mount point against its parent.
+// A plain directory (pre-created before mount) has the same device as its
+// parent; a mounted FUSE filesystem has a different device.
+func (c *dlmTestCluster) waitForMount(mountPoint string, timeout time.Duration) error {
+	parentDir := filepath.Dir(mountPoint)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		parentStat, err := os.Stat(parentDir)
+		if err != nil {
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+		mountStat, err := os.Stat(mountPoint)
+		if err != nil {
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+		parentSys := parentStat.Sys().(*syscall.Stat_t)
+		mountSys := mountStat.Sys().(*syscall.Stat_t)
+		if parentSys.Dev != mountSys.Dev {
+			// Different device = FUSE mounted
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("mount point %s not ready within timeout (FUSE not detected)", mountPoint)
+}
+
+func (c *dlmTestCluster) filerGRPCAddress(idx int) string {
+	return fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[idx])
+}
+
+func (c *dlmTestCluster) waitForFilerCount(expected int, timeout time.Duration) error {
+	addr := fmt.Sprintf("127.0.0.1:%d", c.masterGrpcPort)
+	conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	client := master_pb.NewSeaweedClient(conn)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
+			ClientType: "filer",
+			FilerGroup: filerGroup,
+		})
+		cancel()
+		if err == nil && len(resp.ClusterNodes) >= expected {
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("timed out waiting for %d filers in group %q", expected, filerGroup)
+}
+
+// waitForLockRingConverged verifies that both filers have a consistent view of
+// the lock ring by acquiring the same lock through each filer and checking
+// mutual exclusion. Adapted from test/s3/distributed_lock/.
+func (c *dlmTestCluster) waitForLockRingConverged(timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+
+	owners := []pb.ServerAddress{
+		pb.ServerAddress(c.filerGRPCAddress(0)),
+		pb.ServerAddress(c.filerGRPCAddress(1)),
+	}
+	ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	ring.SetServers(owners)
+
+	attempt := 0
+	for time.Now().Before(deadline) {
+		testKeys := convergenceKeysPerPrimary(ring, owners, attempt)
+		attempt++
+
+		allConverged := true
+		for _, key := range testKeys {
+			converged, _ := c.checkLockMutualExclusion(key)
+			if !converged {
+				allConverged = false
+				break
+			}
+		}
+		if allConverged {
+			return nil
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("lock ring did not converge")
+}
+
+// convergenceKeysPerPrimary generates one test key per primary filer.
+func convergenceKeysPerPrimary(ring *lock_manager.HashRing, owners []pb.ServerAddress, attempt int) []string {
+	found := make(map[pb.ServerAddress]bool)
+	var keys []string
+	for i := 0; len(found) < len(owners) && i < 10000; i++ {
+		key := fmt.Sprintf("convergence-test-%d-%d", attempt, i)
+		primary, _ := ring.GetPrimaryAndBackup(key)
+		if !found[primary] {
+			found[primary] = true
+			keys = append(keys, key)
+		}
+	}
+	return keys
+}
+
+func (c *dlmTestCluster) checkLockMutualExclusion(key string) (bool, error) {
+	// Try to lock via filer0
+	conn0, err := grpc.NewClient(c.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false, err
+	}
+	defer conn0.Close()
+
+	client0 := filer_pb.NewSeaweedFilerClient(conn0)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	resp0, err := client0.DistributedLock(ctx, &filer_pb.LockRequest{
+		Name:          key,
+		SecondsToLock: 5,
+		Owner:         "convergence-test-0",
+	})
+	if err != nil {
+		return false, err
+	}
+	if resp0.Error != "" {
+		return false, fmt.Errorf("lock0: %s", resp0.Error)
+	}
+
+	// Try to lock via filer1 — should fail (already locked)
+	conn1, err := grpc.NewClient(c.filerGRPCAddress(1), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false, err
+	}
+	defer conn1.Close()
+
+	client1 := filer_pb.NewSeaweedFilerClient(conn1)
+	resp1, err := client1.DistributedLock(ctx, &filer_pb.LockRequest{
+		Name:          key,
+		SecondsToLock: 5,
+		Owner:         "convergence-test-1",
+	})
+
+	// Unlock via filer0
+	client0.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
+		Name:       key,
+		RenewToken: resp0.RenewToken,
+	})
+
+	if err != nil {
+		return false, err
+	}
+	// If filer1 also got the lock, the ring hasn't converged
+	if resp1.Error == "" {
+		// Unlock the second one too
+		client1.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
+			Name:       key,
+			RenewToken: resp1.RenewToken,
+		})
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func stopCmd(cmd *exec.Cmd) {
+	if cmd == nil || cmd.Process == nil {
+		return
+	}
+	cmd.Process.Signal(syscall.SIGTERM)
+	done := make(chan struct{})
+	go func() {
+		cmd.Wait()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		cmd.Process.Kill()
+		<-done
+	}
+}
+
+func allocatePorts(t testing.TB, n int) []int {
+	t.Helper()
+	// Hold all listeners open until all ports are collected, then close
+	// them together. This prevents the OS from reassigning a just-freed
+	// port to the next Listen call within the same allocation batch.
+	listeners := make([]net.Listener, 0, n)
+	ports := make([]int, 0, n)
+	defer func() {
+		for _, l := range listeners {
+			l.Close()
+		}
+	}()
+	for i := 0; i < n; i++ {
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(t, err)
+		listeners = append(listeners, l)
+		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
+	}
+	return ports
+}
+
+func findWeedBinary() string {
+	if p := os.Getenv("WEED_BINARY"); p != "" {
+		return p
+	}
+	if p, err := exec.LookPath("weed"); err == nil {
+		return p
+	}
+	candidates := []string{
+		"../../weed/weed",
+		"./weed",
+	}
+	for _, c := range candidates {
+		if info, err := os.Stat(c); err == nil && !info.IsDir() {
+			abs, _ := filepath.Abs(c)
+			return abs
+		}
+	}
+	return ""
+}
@@ -82,12 +82,13 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
 	}
 }

-// freePort asks the OS for a free TCP port.
+// freePort asks the OS for a free TCP port in a range where the gRPC
+// offset (port + 10000) won't collide with well-known ports.
 func freePort(t *testing.T) int {
 	t.Helper()
 	const (
 		minServicePort = 20000
-		maxServicePort = 55535 // SeaweedFS gRPC service uses httpPort + 10000.
+		maxServicePort = 55535
 	)

 	portCount := maxServicePort - minServicePort + 1
@@ -14,7 +14,7 @@ require (
 replace github.com/seaweedfs/seaweedfs => ../../

 require (
-	cloud.google.com/go/auth v0.18.1 // indirect
+	cloud.google.com/go/auth v0.18.2 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
@@ -44,23 +44,23 @@ require (
 	github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
 	github.com/aws/aws-sdk-go v1.55.8 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.32.13 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.13 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect
 	github.com/aws/smithy-go v1.24.2 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
@@ -124,8 +124,8 @@ require (
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
-	github.com/googleapis/gax-go/v2 v2.17.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
+	github.com/googleapis/gax-go/v2 v2.19.0 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gorilla/schema v1.4.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -197,7 +197,6 @@ require (
 	github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
 	github.com/rdleal/intervalst v1.5.0 // indirect
 	github.com/relvacode/iso8601 v1.7.0 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rfjakob/eme v1.1.2 // indirect
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
 	github.com/sagikazarmark/locafero v0.11.0 // indirect
@@ -242,30 +241,29 @@ require (
 	go.mongodb.org/mongo-driver v1.17.9 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
-	go.opentelemetry.io/otel v1.40.0 // indirect
-	go.opentelemetry.io/otel/metric v1.40.0 // indirect
-	go.opentelemetry.io/otel/trace v1.40.0 // indirect
+	go.opentelemetry.io/otel v1.42.0 // indirect
+	go.opentelemetry.io/otel/metric v1.42.0 // indirect
+	go.opentelemetry.io/otel/trace v1.42.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/crypto v0.49.0 // indirect
 	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
 	golang.org/x/image v0.38.0 // indirect
-	golang.org/x/net v0.51.0 // indirect
+	golang.org/x/net v0.52.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.42.0 // indirect
 	golang.org/x/term v0.41.0 // indirect
 	golang.org/x/text v0.35.0 // indirect
-	golang.org/x/time v0.14.0 // indirect
-	google.golang.org/api v0.267.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	google.golang.org/api v0.274.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
 	google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	modernc.org/mathutil v1.7.1 // indirect
 	moul.io/http2curl/v2 v2.3.0 // indirect
 	sigs.k8s.io/yaml v1.6.0 // indirect
 	storj.io/common v0.0.0-20251107171817-6221ae45072c // indirect
@@ -13,8 +13,8 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
 cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
-cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
+cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
+cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
@@ -114,12 +114,12 @@ github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
 github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
 github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
 github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
 github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
@@ -130,24 +130,24 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgq
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 h1:foqo/ocQ7WqKwy3FojGtZQJo0FR4vto9qnz9VaumbCo=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
 github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
@@ -385,12 +385,12 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
 github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
-github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
+github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
+github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
-github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
+github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE=
+github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
 github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
 github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
@@ -600,8 +600,6 @@ github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6ha
 github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
 github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGrdo=
 github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
 github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -754,16 +752,16 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
-go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
-go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
-go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
-go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
-go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
-go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
-go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
-go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
-go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
-go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
+go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
+go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
+go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
+go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
+go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
+go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
+go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
+go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
+go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
+go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
@@ -869,8 +867,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
-golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -981,8 +979,8 @@ golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
-golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -1054,8 +1052,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
-google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
+google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
+google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1091,12 +1089,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
-google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
-google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
-google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
+google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
+google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=
+google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1154,8 +1152,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
-modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
 moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs=
 moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
@@ -11,7 +11,7 @@
    <properties>
        <maven.compiler.source>11</maven.compiler.source>
        <maven.compiler.target>11</maven.compiler.target>
-        <kafka.version>3.9.1</kafka.version>
+        <kafka.version>3.9.2</kafka.version>
        <confluent.version>7.6.0</confluent.version>
    </properties>

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -17,6 +16,8 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )

 const (
@@ -51,8 +52,8 @@ type MasterCluster struct {

 // clusterStatus is the JSON returned by /cluster/status.
 type clusterStatus struct {
-	IsLeader bool   `json:"IsLeader"`
-	Leader   string `json:"Leader"`
+	IsLeader bool     `json:"IsLeader"`
+	Leader   string   `json:"Leader"`
 	Peers    []string `json:"Peers"`
 }

@@ -73,23 +74,23 @@ func StartMasterCluster(t testing.TB) *MasterCluster {
 	logsDir := filepath.Join(baseDir, "logs")
 	os.MkdirAll(logsDir, 0o755)

-	// Allocate 3 port pairs (http, grpc) atomically to prevent reuse.
-	portPairs, err := allocateMultipleMasterPortPairs(3)
+	// Allocate 3 mini-safe ports (each guarantees port+10000 is also free).
+	httpPorts, err := testutil.AllocateMiniPorts(3)
 	if err != nil {
 		t.Fatalf("allocate ports: %v", err)
 	}
 	var nodes [3]*masterNode
 	var peerParts []string
-	for i, pp := range portPairs {
+	for i, hp := range httpPorts {
 		dataDir := filepath.Join(baseDir, fmt.Sprintf("m%d", i))
 		os.MkdirAll(dataDir, 0o755)
 		nodes[i] = &masterNode{
-			port:     pp[0],
-			grpcPort: pp[1],
+			port:     hp,
+			grpcPort: hp + testutil.GrpcPortOffset,
 			dataDir:  dataDir,
 			logFile:  filepath.Join(logsDir, fmt.Sprintf("master%d.log", i)),
 		}
-		peerParts = append(peerParts, fmt.Sprintf("127.0.0.1:%d", pp[0]))
+		peerParts = append(peerParts, fmt.Sprintf("127.0.0.1:%d", hp))
 	}

 	mc := &MasterCluster{
@@ -357,42 +358,6 @@ func (mc *MasterCluster) tailLog(i int) string {
 	return strings.Join(lines, "\n")
 }

-// --- port and binary helpers (adapted from test/volume_server/framework) ---
-
-// allocateMultipleMasterPortPairs finds n non-overlapping (http, grpc) port
-// pairs, holding all listeners until all are found, then releasing them
-// together to avoid races between consecutive allocations.
-func allocateMultipleMasterPortPairs(n int) ([][2]int, error) {
-	var listeners []net.Listener
-	var pairs [][2]int
-
-	defer func() {
-		for _, l := range listeners {
-			l.Close()
-		}
-	}()
-
-	for masterPort := 10000; masterPort <= 55535 && len(pairs) < n; masterPort++ {
-		grpcPort := masterPort + 10000
-		l1, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterPort)))
-		if err != nil {
-			continue
-		}
-		l2, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(grpcPort)))
-		if err != nil {
-			l1.Close()
-			continue
-		}
-		listeners = append(listeners, l1, l2)
-		pairs = append(pairs, [2]int{masterPort, grpcPort})
-	}
-
-	if len(pairs) < n {
-		return nil, fmt.Errorf("could only allocate %d of %d master port pairs", len(pairs), n)
-	}
-	return pairs, nil
-}
-
 func findOrBuildWeedBinary() (string, error) {
 	if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" {
 		if isExecutableFile(fromEnv) {
@@ -0,0 +1,28 @@
+FROM chrislusf/seaweedfs:e2e
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    --no-install-recommends \
+    --no-install-suggests \
+    autoconf \
+    automake \
+    build-essential \
+    git \
+    libtap-harness-archive-perl \
+    perl \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG PJDFSTEST_REPO=https://github.com/sanwan/pjdfstest.git
+ARG PJDFSTEST_REF=d25636a227606f8960e5179741d8f4ad7030ef41
+
+RUN git clone "${PJDFSTEST_REPO}" /opt/pjdfstest && \
+    cd /opt/pjdfstest && \
+    git checkout "${PJDFSTEST_REF}" && \
+    autoreconf -ifs && \
+    ./configure && \
+    make pjdfstest
+
+COPY run_inside_container.sh /run.sh
+COPY known_failures.txt /known_failures.txt
+RUN chmod +x /run.sh
@@ -0,0 +1,57 @@
+services:
+  master:
+    image: chrislusf/seaweedfs:e2e
+    command: "-v=4 master -ip=master -ip.bind=0.0.0.0 -raftBootstrap"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz"]
+      interval: 2s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
+
+  volume:
+    image: chrislusf/seaweedfs:e2e
+    command: "-v=4 volume -master=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "-I", "http://localhost:8080/healthz"]
+      interval: 2s
+      timeout: 10s
+      retries: 15
+      start_period: 5s
+    depends_on:
+      master:
+        condition: service_healthy
+
+  filer:
+    image: chrislusf/seaweedfs:e2e
+    command: "-v=4 filer -master=master:9333 -ip=filer -ip.bind=0.0.0.0"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "-I", "http://localhost:8888/healthz"]
+      interval: 2s
+      timeout: 10s
+      retries: 15
+      start_period: 5s
+    depends_on:
+      volume:
+        condition: service_healthy
+
+  mount:
+    image: chrislusf/seaweedfs:pjdfstest
+    build:
+      context: .
+    command: "-v=4 mount -filer=filer:8888 -filer.path=/ -dirAutoCreate -dir=/mnt/seaweedfs -allowOthers"
+    cap_add:
+      - SYS_ADMIN
+    devices:
+      - /dev/fuse
+    security_opt:
+      - apparmor:unconfined
+    healthcheck:
+      test: ["CMD", "mountpoint", "-q", "--", "/mnt/seaweedfs"]
+      interval: 2s
+      timeout: 10s
+      retries: 15
+      start_period: 10s
+    depends_on:
+      filer:
+        condition: service_healthy
@@ -0,0 +1,22 @@
+# Known pjdfstest failures for SeaweedFS FUSE mount.
+#
+# Tests listed here are skipped during CI runs. Each entry must be a path
+# relative to the pjdfstest root (e.g. tests/chmod/02.t).
+#
+# A failure in any test NOT listed here will cause the CI job to fail,
+# catching regressions immediately.
+
+# ── Directory rename permission edge case ──────────────────────────────
+# Cross-directory rename of a subdirectory with restricted permissions
+# causes cascading test failures within the test file.
+tests/rename/21.t
+
+# ── Hard link nlink count inconsistencies ────────────────────────────
+# link/00.t and unlink/00.t fail nlink assertions (e.g. expected nlink=2,
+# got nlink=3) after hard link creation/removal. This is a filer-side hard
+# link counter issue, not a mount mtime/ctime problem. The failures are
+# deterministic and surfaced by caching changes that affect the order in
+# which entries are loaded into the local meta cache.
+tests/link/00.t
+tests/unlink/00.t
+
@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+#
+# Run the pjdfstest POSIX compliance suite against a SeaweedFS FUSE mount.
+#
+# This script:
+#   1. Starts a self-contained "weed mini" server (master+volume+filer in one)
+#   2. Mounts the filesystem with "weed mount"
+#   3. Builds pjdfstest from upstream and runs it under prove(1)
+#
+# Requirements: weed in $PATH, fusermount3, perl with TAP::Harness::Archive,
+# autoconf, make, sudo (pjdfstest exercises chown/chmod which need root).
+#
+# Usage:
+#   test/pjdfstest/run.sh                # runs full suite
+#   PJDFSTEST_TESTS=tests/chmod ./run.sh # runs a subset
+
+set -euo pipefail
+
+WEED_BIN="${WEED_BIN:-weed}"
+WORK_DIR="${WORK_DIR:-$(mktemp -d /tmp/seaweedfs-pjdfstest.XXXXXX)}"
+MOUNT_DIR="${MOUNT_DIR:-${WORK_DIR}/mnt}"
+DATA_DIR="${DATA_DIR:-${WORK_DIR}/data}"
+LOG_DIR="${LOG_DIR:-${WORK_DIR}/logs}"
+FILER_PORT="${FILER_PORT:-28888}"
+FILER_ADDR="127.0.0.1:${FILER_PORT}"
+
+# Pin to an immutable upstream commit so CI is reproducible. Override via env
+# if you want to test against a different ref or fork.
+PJDFSTEST_REPO="${PJDFSTEST_REPO:-https://github.com/sanwan/pjdfstest.git}"
+PJDFSTEST_REF="${PJDFSTEST_REF:-d25636a227606f8960e5179741d8f4ad7030ef41}"
+PJDFSTEST_TESTS="${PJDFSTEST_TESTS:-tests/}"
+
+mini_pid=""
+mount_pid=""
+
+CI_LOG_DIR="/tmp/seaweedfs-pjdfstest-logs"
+
+cleanup() {
+  set +e
+  if [[ -n "${mount_pid}" ]] && kill -0 "${mount_pid}" 2>/dev/null; then
+    kill -TERM "${mount_pid}" 2>/dev/null || true
+    wait "${mount_pid}" 2>/dev/null || true
+  fi
+  if mountpoint -q "${MOUNT_DIR}" 2>/dev/null; then
+    fusermount3 -u "${MOUNT_DIR}" 2>/dev/null || \
+      fusermount -u "${MOUNT_DIR}" 2>/dev/null || \
+      sudo umount "${MOUNT_DIR}" 2>/dev/null || true
+  fi
+  if [[ -n "${mini_pid}" ]] && kill -0 "${mini_pid}" 2>/dev/null; then
+    kill -TERM "${mini_pid}" 2>/dev/null || true
+    wait "${mini_pid}" 2>/dev/null || true
+  fi
+  # Copy logs to a fixed path for CI artifact upload.
+  mkdir -p "${CI_LOG_DIR}"
+  cp "${LOG_DIR}"/*.log "${CI_LOG_DIR}/" 2>/dev/null || true
+}
+trap cleanup EXIT INT TERM
+
+mkdir -p "${MOUNT_DIR}" "${DATA_DIR}" "${LOG_DIR}"
+
+echo "==> Starting weed mini on ${FILER_ADDR}"
+"${WEED_BIN}" mini \
+  -dir="${DATA_DIR}" \
+  -ip=127.0.0.1 \
+  -filer.port="${FILER_PORT}" \
+  -s3=false \
+  -webdav=false \
+  -admin.ui=false \
+  >"${LOG_DIR}/mini.log" 2>&1 &
+mini_pid=$!
+
+# Wait for filer to accept connections.
+for i in $(seq 1 60); do
+  if (echo > "/dev/tcp/127.0.0.1/${FILER_PORT}") 2>/dev/null; then
+    break
+  fi
+  if ! kill -0 "${mini_pid}" 2>/dev/null; then
+    echo "weed mini exited early; log tail:" >&2
+    tail -n 100 "${LOG_DIR}/mini.log" >&2 || true
+    exit 1
+  fi
+  sleep 0.5
+done
+
+if ! (echo > "/dev/tcp/127.0.0.1/${FILER_PORT}") 2>/dev/null; then
+  echo "weed mini filer did not become reachable within 30s; log tail:" >&2
+  tail -n 100 "${LOG_DIR}/mini.log" >&2 || true
+  exit 1
+fi
+
+echo "==> Mounting SeaweedFS at ${MOUNT_DIR}"
+# allowOthers is required so that pjdfstest's setuid/setgid sub-tests (run via
+# sudo) can access files created as the invoking user.
+sudo "${WEED_BIN}" mount \
+  -filer="${FILER_ADDR}" \
+  -dir="${MOUNT_DIR}" \
+  -filer.path=/ \
+  -dirAutoCreate \
+  -allowOthers=true \
+  >"${LOG_DIR}/mount.log" 2>&1 &
+mount_pid=$!
+
+for i in $(seq 1 60); do
+  if mountpoint -q "${MOUNT_DIR}"; then
+    break
+  fi
+  if ! kill -0 "${mount_pid}" 2>/dev/null; then
+    echo "weed mount exited early; log tail:" >&2
+    tail -n 100 "${LOG_DIR}/mount.log" >&2 || true
+    exit 1
+  fi
+  sleep 0.5
+done
+
+if ! mountpoint -q "${MOUNT_DIR}"; then
+  echo "FUSE mount did not come up within 30s" >&2
+  tail -n 100 "${LOG_DIR}/mount.log" >&2 || true
+  exit 1
+fi
+
+echo "==> Cloning and building pjdfstest"
+PJDFSTEST_DIR="${WORK_DIR}/pjdfstest"
+if [[ ! -d "${PJDFSTEST_DIR}/.git" ]]; then
+  git clone "${PJDFSTEST_REPO}" "${PJDFSTEST_DIR}"
+fi
+git -C "${PJDFSTEST_DIR}" remote set-url origin "${PJDFSTEST_REPO}"
+git -C "${PJDFSTEST_DIR}" fetch --depth 1 origin "${PJDFSTEST_REF}"
+git -C "${PJDFSTEST_DIR}" checkout --detach FETCH_HEAD
+(
+  cd "${PJDFSTEST_DIR}"
+  autoreconf -ifs
+  ./configure
+  make pjdfstest
+)
+
+# pjdfstest must run inside the filesystem under test.
+TEST_ROOT="${MOUNT_DIR}/pjdfstest-root"
+sudo mkdir -p "${TEST_ROOT}"
+sudo cp -r "${PJDFSTEST_DIR}/." "${TEST_ROOT}/"
+
+cd "${TEST_ROOT}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+KNOWN_FAILURES="${SCRIPT_DIR}/known_failures.txt"
+
+# Build the list of tests to run, excluding known failures.
+if [[ -f "${KNOWN_FAILURES}" ]] && [[ "${PJDFSTEST_TESTS}" == "tests/" ]]; then
+  mapfile -t skip < <(grep -v '^#' "${KNOWN_FAILURES}" | grep -v '^$')
+  all_tests=()
+  while IFS= read -r -d '' t; do
+    all_tests+=("$t")
+  done < <(find tests/ -name '*.t' -print0 | sort -z)
+
+  run_tests=()
+  for t in "${all_tests[@]}"; do
+    is_skipped=false
+    for s in "${skip[@]}"; do
+      if [[ "$t" == "$s" ]]; then
+        is_skipped=true
+        break
+      fi
+    done
+    if ! $is_skipped; then
+      run_tests+=("$t")
+    fi
+  done
+
+  echo "==> Running pjdfstest (${#run_tests[@]} tests, ${#skip[@]} skipped)"
+  sudo prove -rv "${run_tests[@]}"
+else
+  echo "==> Running pjdfstest (${PJDFSTEST_TESTS})"
+  sudo prove -rv "${PJDFSTEST_TESTS}"
+fi
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+#
+# Runs pjdfstest inside the mount container against /mnt/seaweedfs.
+# Invoked via: docker compose exec mount /run.sh
+#
+# Uses known_failures.txt to skip tests with known issues. Any failure
+# in a test NOT in the skip list causes a non-zero exit (regression).
+set -euo pipefail
+
+MOUNT_DIR="/mnt/seaweedfs"
+PJDFSTEST_TESTS="${PJDFSTEST_TESTS:-tests/}"
+KNOWN_FAILURES="/known_failures.txt"
+
+# Copy pjdfstest into the mounted filesystem so tests exercise the FS under test.
+TEST_ROOT="${MOUNT_DIR}/pjdfstest-root"
+mkdir -p "${TEST_ROOT}"
+cp -r /opt/pjdfstest/. "${TEST_ROOT}/"
+
+cd "${TEST_ROOT}"
+
+# Build the list of tests to run, excluding known failures.
+if [[ -f "${KNOWN_FAILURES}" ]] && [[ "${PJDFSTEST_TESTS}" == "tests/" ]]; then
+  mapfile -t skip < <(grep -v '^#' "${KNOWN_FAILURES}" | grep -v '^$')
+  all_tests=()
+  while IFS= read -r -d '' t; do
+    all_tests+=("$t")
+  done < <(find tests/ -name '*.t' -print0 | sort -z)
+
+  run_tests=()
+  for t in "${all_tests[@]}"; do
+    is_skipped=false
+    for s in "${skip[@]}"; do
+      if [[ "$t" == "$s" ]]; then
+        is_skipped=true
+        break
+      fi
+    done
+    if ! $is_skipped; then
+      run_tests+=("$t")
+    fi
+  done
+
+  echo "==> Running pjdfstest (${#run_tests[@]} tests, ${#skip[@]} skipped)"
+  prove -rv "${run_tests[@]}"
+else
+  echo "==> Running pjdfstest (${PJDFSTEST_TESTS})"
+  prove -rv "${PJDFSTEST_TESTS}"
+fi
@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"testing"

+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 )

@@ -21,7 +22,12 @@ func WriteTestVolumeFiles(t *testing.T, baseDir string, volumeID uint32, datSize
 	datPath := filepath.Join(baseDir, volumeFilename(volumeID, ".dat"))
 	idxPath := filepath.Join(baseDir, volumeFilename(volumeID, ".idx"))

-	data := make([]byte, datSize)
+	// The idx entry records the needle's data size. The actual on-disk size
+	// includes header + checksum + timestamp (GetActualSize). The .dat must
+	// be large enough to hold the full needle.
+	needleDataSize := types.Size(datSize)
+	actualSize := needle.GetActualSize(needleDataSize, needle.Version3)
+	data := make([]byte, actualSize)
 	rng := rand.New(rand.NewSource(99))
 	_, _ = rng.Read(data)
 	if err := os.WriteFile(datPath, data, 0644); err != nil {
@@ -35,7 +41,7 @@ func WriteTestVolumeFiles(t *testing.T, baseDir string, volumeID uint32, datSize

 	types.NeedleIdToBytes(entry[:idEnd], types.NeedleId(1))
 	types.OffsetToBytes(entry[idEnd:offsetEnd], types.ToOffset(0))
-	types.SizeToBytes(entry[offsetEnd:sizeEnd], types.Size(datSize))
+	types.SizeToBytes(entry[offsetEnd:sizeEnd], needleDataSize)

 	if err := os.WriteFile(idxPath, entry, 0644); err != nil {
 		t.Fatalf("write idx file: %v", err)
@@ -5,7 +5,6 @@ import (
 	"crypto/rand"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -13,6 +12,8 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )

 type TestEnvironment struct {
@@ -96,29 +97,26 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 		t.Fatalf("Failed to create temp dir: %v", err)
 	}

-	bindIP := findBindIP()
+	bindIP := testutil.FindBindIP()

-	masterPort, masterGrpcPort := mustFreePortPair(t, "Master")
-	volumePort, volumeGrpcPort := mustFreePortPair(t, "Volume")
-	filerPort, filerGrpcPort := mustFreePortPair(t, "Filer")
-	s3Port, s3GrpcPort := mustFreePortPair(t, "S3")
-	icebergPort := mustFreePort(t, "Iceberg")
+	// 9 ports: master(2), volume(2), filer(2), s3(2), iceberg(1)
+	ports := testutil.MustAllocatePorts(t, 9)

 	return &TestEnvironment{
 		seaweedDir:      seaweedDir,
 		weedBinary:      weedBinary,
 		dataDir:         dataDir,
 		bindIP:          bindIP,
-		s3Port:          s3Port,
-		s3GrpcPort:      s3GrpcPort,
-		icebergPort:     icebergPort,
-		masterPort:      masterPort,
-		masterGrpcPort:  masterGrpcPort,
-		filerPort:       filerPort,
-		filerGrpcPort:   filerGrpcPort,
-		volumePort:      volumePort,
-		volumeGrpcPort:  volumeGrpcPort,
-		dockerAvailable: hasDocker(),
+		masterPort:      ports[0],
+		masterGrpcPort:  ports[1],
+		volumePort:      ports[2],
+		volumeGrpcPort:  ports[3],
+		filerPort:       ports[4],
+		filerGrpcPort:   ports[5],
+		s3Port:          ports[6],
+		s3GrpcPort:      ports[7],
+		icebergPort:     ports[8],
+		dockerAvailable: testutil.HasDocker(),
 	}
 }

@@ -333,73 +331,6 @@ func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) {
 	}
 }

-func hasDocker() bool {
-	cmd := exec.Command("docker", "version")
-	return cmd.Run() == nil
-}
-
-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-
-	port, err := getFreePort()
-	if err != nil {
-		t.Fatalf("Failed to get free port for %s: %v", name, err)
-	}
-	return port
-}
-
-func mustFreePortPair(t *testing.T, name string) (int, int) {
-	t.Helper()
-
-	httpPort, grpcPort, err := findAvailablePortPair()
-	if err != nil {
-		t.Fatalf("Failed to get free port pair for %s: %v", name, err)
-	}
-	return httpPort, grpcPort
-}
-
-func findAvailablePortPair() (int, int, error) {
-	httpPort, err := getFreePort()
-	if err != nil {
-		return 0, 0, err
-	}
-	grpcPort, err := getFreePort()
-	if err != nil {
-		return 0, 0, err
-	}
-	return httpPort, grpcPort, nil
-}
-
-func getFreePort() (int, error) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		return 0, err
-	}
-	defer listener.Close()
-
-	addr := listener.Addr().(*net.TCPAddr)
-	return addr.Port, nil
-}
-
-func findBindIP() string {
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		return "127.0.0.1"
-	}
-	for _, addr := range addrs {
-		ipNet, ok := addr.(*net.IPNet)
-		if !ok || ipNet.IP == nil {
-			continue
-		}
-		ip := ipNet.IP.To4()
-		if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
-			continue
-		}
-		return ip.String()
-	}
-	return "127.0.0.1"
-}
-
 func randomString(length int) string {
 	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
 	b := make([]byte, length)
@@ -20,6 +20,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
 	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -115,7 +116,7 @@ func startDistributedLockCluster(t *testing.T) *distributedLockCluster {
 		require.NoError(t, os.MkdirAll(dir, 0o755), "create %s", dir)
 	}

-	ports, err := allocatePorts(12)
+	ports, err := testutil.AllocatePorts(12)
 	require.NoError(t, err, "allocate ports")
 	cluster.masterPort = ports[0]
 	cluster.masterGrpcPort = ports[1]
@@ -591,26 +592,6 @@ func (c *distributedLockCluster) tailLog(name string) string {
 	return strings.Join(lines, "\n")
 }

-func allocatePorts(count int) ([]int, error) {
-	listeners := make([]net.Listener, 0, count)
-	ports := make([]int, 0, count)
-	for i := 0; i < count; i++ {
-		l, err := net.Listen("tcp", "127.0.0.1:0")
-		if err != nil {
-			for _, openListener := range listeners {
-				_ = openListener.Close()
-			}
-			return nil, err
-		}
-		listeners = append(listeners, l)
-		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
-	}
-	for _, l := range listeners {
-		_ = l.Close()
-	}
-	return ports, nil
-}
-
 func stopProcess(cmd *exec.Cmd) {
 	if cmd == nil || cmd.Process == nil {
 		return
@@ -321,6 +321,13 @@ func TestIAMPolicyManagement(t *testing.T) {
 		require.NoError(t, err)
 		defer iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})

+		// List user policies before any inline policy is attached
+		listResp, err := iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.Empty(t, listResp.PolicyNames, "New user should have no inline policies")
+
 		policyName := "test-inline-policy"
 		policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::*"}]}`

@@ -332,6 +339,14 @@ func TestIAMPolicyManagement(t *testing.T) {
 		})
 		require.NoError(t, err)

+		// List user policies after attaching inline policy
+		listResp, err = iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.NotEmpty(t, listResp.PolicyNames, "User should have inline policies after PutUserPolicy")
+		assert.False(t, *listResp.IsTruncated)
+
 		// Get user policy
 		getResp, err := iamClient.GetUserPolicy(&iam.GetUserPolicyInput{
 			UserName:   aws.String(userName),
@@ -348,5 +363,22 @@ func TestIAMPolicyManagement(t *testing.T) {
 			PolicyName: aws.String(policyName),
 		})
 		require.NoError(t, err)
+
+		// List user policies after deletion
+		listResp, err = iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.Empty(t, listResp.PolicyNames, "User should have no inline policies after DeleteUserPolicy")
+	})
+
+	t.Run("list_user_policies_nonexistent_user", func(t *testing.T) {
+		_, err := iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
+			UserName: aws.String("nonexistent-user-for-list-policies"),
+		})
+		require.Error(t, err)
+		awsErr, ok := err.(awserr.Error)
+		require.True(t, ok, "Expected AWS error type")
+		assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
 	})
 }
@@ -778,6 +778,105 @@ func TestIAMGroupRawAPI(t *testing.T) {
 	})
 }

+// TestIAMGroupInlinePolicy tests group inline policy operations:
+// PutGroupPolicy, GetGroupPolicy, ListGroupPolicies, DeleteGroupPolicy
+func TestIAMGroupInlinePolicy(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	// Skip if running against embedded IAM (which returns NotImplemented for group inline policies)
+	_, probeErr := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{GroupName: aws.String("probe-group-inline-support")})
+	if probeErr != nil {
+		if awsErr, ok := probeErr.(awserr.Error); ok && awsErr.Code() == "NotImplemented" {
+			t.Skip("Skipping: group inline policies not supported in embedded IAM mode")
+		}
+	}
+	require.NoError(t, err)
+
+	groupName := "test-group-inline-policy"
+	policyName := "TestInlinePolicy"
+	_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
+		GroupName: aws.String(groupName),
+	})
+	require.NoError(t, err)
+	defer func() {
+		// Clean up inline policies before deleting the group
+		iamClient.DeleteGroupPolicy(&iam.DeleteGroupPolicyInput{
+			GroupName:  aws.String(groupName),
+			PolicyName: aws.String(policyName),
+		})
+		iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)})
+	}()
+
+	t.Run("list_empty", func(t *testing.T) {
+		resp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		assert.Empty(t, resp.PolicyNames)
+		assert.False(t, *resp.IsTruncated)
+	})
+
+	policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}`
+
+	t.Run("put_group_policy", func(t *testing.T) {
+		_, err := iamClient.PutGroupPolicy(&iam.PutGroupPolicyInput{
+			GroupName:      aws.String(groupName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(policyDoc),
+		})
+		require.NoError(t, err)
+	})
+
+	t.Run("list_after_put", func(t *testing.T) {
+		resp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		assert.NotEmpty(t, resp.PolicyNames)
+	})
+
+	t.Run("get_group_policy", func(t *testing.T) {
+		resp, err := iamClient.GetGroupPolicy(&iam.GetGroupPolicyInput{
+			GroupName:  aws.String(groupName),
+			PolicyName: aws.String(policyName),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, groupName, *resp.GroupName)
+		assert.Equal(t, policyName, *resp.PolicyName)
+		assert.Contains(t, *resp.PolicyDocument, "s3:GetObject")
+	})
+
+	t.Run("delete_group_policy", func(t *testing.T) {
+		_, err := iamClient.DeleteGroupPolicy(&iam.DeleteGroupPolicyInput{
+			GroupName:  aws.String(groupName),
+			PolicyName: aws.String(policyName),
+		})
+		require.NoError(t, err)
+	})
+
+	t.Run("list_after_delete", func(t *testing.T) {
+		resp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		assert.Empty(t, resp.PolicyNames)
+	})
+
+	t.Run("nonexistent_group", func(t *testing.T) {
+		_, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String("nonexistent-group-for-policies"),
+		})
+		require.Error(t, err)
+		awsErr, ok := err.(awserr.Error)
+		require.True(t, ok, "Expected AWS error type")
+		assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+}
+
 // createS3Client creates an S3 client with static credentials
 func createS3Client(t *testing.T, accessKey, secretKey string) *s3.S3 {
 	sess, err := session.NewSession(&aws.Config{
@@ -7,7 +7,6 @@ import (
 	"encoding/base64"
 	"fmt"
 	"math/rand"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -25,6 +24,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"

+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
 	"github.com/seaweedfs/seaweedfs/weed/command"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -39,19 +39,19 @@ const (

 // TestCluster manages the weed mini instance for integration testing
 type TestCluster struct {
-	dataDir        string
-	ctx            context.Context
-	cancel         context.CancelFunc
-	s3Client       *s3.S3
-	isRunning      bool
-	startOnce      sync.Once
-	wg             sync.WaitGroup
-	masterPort     int
-	volumePort     int
-	filerPort      int
-	s3Port         int
-	s3Endpoint     string
-	rustVolumeCmd  *exec.Cmd
+	dataDir       string
+	ctx           context.Context
+	cancel        context.CancelFunc
+	s3Client      *s3.S3
+	isRunning     bool
+	startOnce     sync.Once
+	wg            sync.WaitGroup
+	masterPort    int
+	volumePort    int
+	filerPort     int
+	s3Port        int
+	s3Endpoint    string
+	rustVolumeCmd *exec.Cmd
 }

 // TestS3Integration demonstrates basic S3 operations against a running weed mini instance
@@ -112,37 +112,16 @@ func TestS3Integration(t *testing.T) {
 }

 // findAvailablePort finds an available port by binding to port 0
-func findAvailablePort() (int, error) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		return 0, err
-	}
-	defer listener.Close()
-
-	addr := listener.Addr().(*net.TCPAddr)
-	return addr.Port, nil
-}

 // startMiniCluster starts a weed mini instance directly without exec.
 // Extra flags (e.g. "-s3.allowDeleteBucketNotEmpty=false") can be appended via extraArgs.
 func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
-	// Find available ports
-	masterPort, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find master port: %v", err)
-	}
-	volumePort, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find volume port: %v", err)
-	}
-	filerPort, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find filer port: %v", err)
-	}
-	s3Port, err := findAvailablePort()
-	if err != nil {
-		return nil, fmt.Errorf("failed to find s3 port: %v", err)
-	}
+	// Allocate non-colliding ports (including gRPC offsets) for weed mini.
+	ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3"})
+	masterPort := ports[0]
+	volumePort := ports[1]
+	filerPort := ports[2]
+	s3Port := ports[3]
 	// Create temporary directory for test data
 	testDir := t.TempDir()

@@ -167,7 +146,7 @@ func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {

 	// Create empty security.toml to disable JWT authentication in tests
 	securityToml := filepath.Join(testDir, "security.toml")
-	err = os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
+	err := os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
 	if err != nil {
 		cancel()
 		return nil, fmt.Errorf("failed to create security.toml: %v", err)
@@ -233,10 +212,9 @@ func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
 	}()

 	// Wait for S3 service to be ready
-	err = waitForS3Ready(cluster.s3Endpoint, 30*time.Second)
-	if err != nil {
+	if !testutil.WaitForService(cluster.s3Endpoint, 30*time.Second) {
 		cancel()
-		return nil, fmt.Errorf("S3 service failed to start: %v", err)
+		return nil, fmt.Errorf("S3 service failed to start at %s", cluster.s3Endpoint)
 	}

 	// If VOLUME_SERVER_IMPL=rust, start a Rust volume server alongside weed mini
@@ -277,14 +255,12 @@ func (c *TestCluster) startRustVolumeServer(t *testing.T) error {
 		return fmt.Errorf("resolve rust volume binary: %v", err)
 	}

-	rustVolumePort, err := findAvailablePort()
+	rustPorts, err := testutil.AllocatePorts(2)
 	if err != nil {
-		return fmt.Errorf("find rust volume port: %v", err)
-	}
-	rustVolumeGrpcPort, err := findAvailablePort()
-	if err != nil {
-		return fmt.Errorf("find rust volume grpc port: %v", err)
+		return fmt.Errorf("find rust volume ports: %v", err)
 	}
+	rustVolumePort := rustPorts[0]
+	rustVolumeGrpcPort := rustPorts[1]

 	rustVolumeDir := filepath.Join(c.dataDir, "rust-volume")
 	if err := os.MkdirAll(rustVolumeDir, 0o755); err != nil {
@@ -377,25 +353,6 @@ func (c *TestCluster) Stop() {
 	}
 }

-// waitForS3Ready waits for the S3 service to be ready
-func waitForS3Ready(endpoint string, timeout time.Duration) error {
-	client := &http.Client{Timeout: 1 * time.Second}
-	deadline := time.Now().Add(timeout)
-
-	for time.Now().Before(deadline) {
-		resp, err := client.Get(endpoint)
-		if err == nil {
-			resp.Body.Close()
-			// Wait a bit more to ensure service is fully ready
-			time.Sleep(500 * time.Millisecond)
-			return nil
-		}
-		time.Sleep(200 * time.Millisecond)
-	}
-
-	return fmt.Errorf("timeout waiting for S3 service at %s", endpoint)
-}
-
 // Test functions

 func testCreateBucket(t *testing.T, cluster *TestCluster) {
@@ -317,4 +317,3 @@ func hasKey(policy map[string]interface{}, key string) bool {

 	return false
 }
-
@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -21,6 +20,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
 	"github.com/seaweedfs/seaweedfs/weed/command"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -703,43 +703,12 @@ func uniqueName(prefix string) string {

 // --- Test setup helpers ---

-func findAvailablePort() (int, error) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		return 0, err
-	}
-	defer listener.Close()
-	addr := listener.Addr().(*net.TCPAddr)
-	return addr.Port, nil
-}
-
-// findAvailablePortPair finds an available http port P such that P and P+10000 (grpc) are both available
-func findAvailablePortPair() (int, int, error) {
-	httpPort, err := findAvailablePort()
-	if err != nil {
-		return 0, 0, err
-	}
-	for i := 0; i < 100; i++ {
-		grpcPort, err := findAvailablePort()
-		if err != nil {
-			return 0, 0, err
-		}
-		if grpcPort != httpPort {
-			return httpPort, grpcPort, nil
-		}
-	}
-	return 0, 0, fmt.Errorf("failed to find available port pair")
-}
-
 func startMiniCluster(t *testing.T) (*TestCluster, error) {
-	masterPort, masterGrpcPort, err := findAvailablePortPair()
-	require.NoError(t, err)
-	volumePort, volumeGrpcPort, err := findAvailablePortPair()
-	require.NoError(t, err)
-	filerPort, filerGrpcPort, err := findAvailablePortPair()
-	require.NoError(t, err)
-	s3Port, s3GrpcPort, err := findAvailablePortPair()
-	require.NoError(t, err)
+	ports := testutil.MustAllocatePorts(t, 8)
+	masterPort, masterGrpcPort := ports[0], ports[1]
+	volumePort, volumeGrpcPort := ports[2], ports[3]
+	filerPort, filerGrpcPort := ports[4], ports[5]
+	s3Port, s3GrpcPort := ports[6], ports[7]

 	testDir := t.TempDir()

@@ -760,13 +729,15 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {

 	// Disable authentication for tests
 	securityToml := filepath.Join(testDir, "security.toml")
-	err = os.WriteFile(securityToml, []byte("# Empty security config\n"), 0644)
+	err := os.WriteFile(securityToml, []byte("# Empty security config\n"), 0644)
 	require.NoError(t, err)

-	// Configure credential store for IAM tests
+	// Configure credential store for IAM tests.
+	// Use filer_etc instead of memory because the memory store does not
+	// persist groups or service accounts through LoadConfiguration/SaveConfiguration.
 	credentialToml := filepath.Join(testDir, "credential.toml")
 	credentialConfig := `
-[credential.memory]
+[credential.filer_etc]
 enabled = true
 `
 	err = os.WriteFile(credentialToml, []byte(credentialConfig), 0644)
@@ -819,10 +790,9 @@ enabled = true
 	}()

 	// Wait for S3
-	err = waitForS3Ready(cluster.s3Endpoint, 60*time.Second)
-	if err != nil {
+	if !testutil.WaitForService(cluster.s3Endpoint, 60*time.Second) {
 		cancel()
-		return nil, err
+		return nil, fmt.Errorf("timeout waiting for S3 at %s", cluster.s3Endpoint)
 	}

 	// If VOLUME_SERVER_IMPL=rust, start a Rust volume server alongside weed mini
@@ -837,20 +807,6 @@ enabled = true
 	return cluster, nil
 }

-func waitForS3Ready(endpoint string, timeout time.Duration) error {
-	client := &http.Client{Timeout: 1 * time.Second}
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		resp, err := client.Get(endpoint)
-		if err == nil {
-			resp.Body.Close()
-			return nil
-		}
-		time.Sleep(200 * time.Millisecond)
-	}
-	return fmt.Errorf("timeout waiting for S3")
-}
-
 // startRustVolumeServer starts a Rust volume server that registers with the same master.
 func (c *TestCluster) startRustVolumeServer(t *testing.T) error {
 	t.Helper()
@@ -860,14 +816,12 @@ func (c *TestCluster) startRustVolumeServer(t *testing.T) error {
 		return fmt.Errorf("resolve rust volume binary: %v", err)
 	}

-	rustVolumePort, err := findAvailablePort()
+	rustPorts, err := testutil.AllocatePorts(2)
 	if err != nil {
-		return fmt.Errorf("find rust volume port: %v", err)
-	}
-	rustVolumeGrpcPort, err := findAvailablePort()
-	if err != nil {
-		return fmt.Errorf("find rust volume grpc port: %v", err)
+		return fmt.Errorf("find rust volume ports: %v", err)
 	}
+	rustVolumePort := rustPorts[0]
+	rustVolumeGrpcPort := rustPorts[1]

 	rustVolumeDir := filepath.Join(c.dataDir, "rust-volume")
 	if err := os.MkdirAll(rustVolumeDir, 0o755); err != nil {
@@ -0,0 +1,75 @@
+package policy
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/require"
+)
+
+// TestShellAccessKeyLifecycle exercises s3.accesskey.* commands end-to-end.
+func TestShellAccessKeyLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	const weedCmd = "weed"
+	master := string(pb.NewServerAddress("127.0.0.1", cluster.masterPort, cluster.masterGrpcPort))
+	filer := string(pb.NewServerAddress("127.0.0.1", cluster.filerPort, cluster.filerGrpcPort))
+
+	userName := uniqueName("akuser")
+	// Create user with explicit key so we know the initial value.
+	initialAK := "INITIALAK1234567890X"
+	initialSK := "initialsecret1234567890abcdefghijklmnop"
+	execShell(t, weedCmd, master, filer,
+		fmt.Sprintf("s3.user.create -name %s -access_key %s -secret_key %s", userName, initialAK, initialSK))
+	defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.delete -name %s", userName))
+
+	t.Run("ListInitialKey", func(t *testing.T) {
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.accesskey.list -user %s", userName))
+		requireContains(t, out, initialAK, "accesskey.list initial")
+	})
+
+	var createdAK string
+	t.Run("CreateAdditionalKey", func(t *testing.T) {
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.accesskey.create -user %s", userName))
+		requireContains(t, out, "Access Key:", "accesskey.create output")
+		requireContains(t, out, "Secret Key:", "accesskey.create output")
+		createdAK = extractFieldAfter(out, "Access Key:")
+		if createdAK == "" {
+			t.Fatalf("failed to extract access key from create output:\n%s", out)
+		}
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.accesskey.list -user %s", userName))
+		requireContains(t, out, initialAK, "list contains original")
+		requireContains(t, out, createdAK, "list contains new key")
+	})
+
+	t.Run("RotateKey", func(t *testing.T) {
+		if createdAK == "" {
+			t.Fatal("createdAK is empty; CreateAdditionalKey must run successfully first")
+		}
+		out := execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.accesskey.rotate -user %s -access_key %s", userName, initialAK))
+		requireContains(t, out, initialAK, "rotate shows old key")
+		requireContains(t, out, "deleted", "rotate marks old key deleted")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.accesskey.list -user %s", userName))
+		requireNotContains(t, out, initialAK, "old key removed")
+		requireContains(t, out, createdAK, "other key still present")
+	})
+
+	t.Run("DeleteKey", func(t *testing.T) {
+		if createdAK == "" {
+			t.Fatal("createdAK is empty; CreateAdditionalKey must run successfully first")
+		}
+		execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.accesskey.delete -user %s -access_key %s", userName, createdAK))
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.accesskey.list -user %s", userName))
+		requireNotContains(t, out, createdAK, "deleted key removed from list")
+	})
+}
@@ -0,0 +1,54 @@
+package policy
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/require"
+)
+
+// TestShellAnonymousAccess exercises s3.anonymous.* commands end-to-end.
+func TestShellAnonymousAccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	const weedCmd = "weed"
+	master := string(pb.NewServerAddress("127.0.0.1", cluster.masterPort, cluster.masterGrpcPort))
+	filer := string(pb.NewServerAddress("127.0.0.1", cluster.filerPort, cluster.filerGrpcPort))
+
+	bucketName := uniqueName("anon-bkt")
+	execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.create -name %s", bucketName))
+	defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.delete -name %s", bucketName))
+
+	t.Run("SetAndGet", func(t *testing.T) {
+		out := execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.anonymous.set -bucket %s -access Read,List", bucketName))
+		requireContains(t, out, bucketName, "anonymous.set output")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.anonymous.get -bucket %s", bucketName))
+		requireContains(t, out, bucketName, "anonymous.get bucket")
+		requireContains(t, out, "Read", "anonymous.get read action")
+		requireContains(t, out, "List", "anonymous.get list action")
+	})
+
+	t.Run("List", func(t *testing.T) {
+		out := execShell(t, weedCmd, master, filer, "s3.anonymous.list")
+		requireContains(t, out, bucketName, "anonymous.list contains bucket")
+	})
+
+	t.Run("SetNone", func(t *testing.T) {
+		execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.anonymous.set -bucket %s -access none", bucketName))
+
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.anonymous.get -bucket %s", bucketName))
+		requireContains(t, out, "none", "anonymous.get after set none")
+
+		out = execShell(t, weedCmd, master, filer, "s3.anonymous.list")
+		requireNotContains(t, out, bucketName, "anonymous.list after clearing")
+	})
+}
@@ -0,0 +1,128 @@
+package policy
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/require"
+)
+
+// TestShellBucketLifecycle exercises s3.bucket.* commands end-to-end:
+// create/list/delete, owner, quota, versioning, lock, quota.enforce.
+func TestShellBucketLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	const weedCmd = "weed"
+	master := string(pb.NewServerAddress("127.0.0.1", cluster.masterPort, cluster.masterGrpcPort))
+	filer := string(pb.NewServerAddress("127.0.0.1", cluster.filerPort, cluster.filerGrpcPort))
+
+	t.Run("CreateListDelete", func(t *testing.T) {
+		bucketName := uniqueName("bkt")
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.create -name %s", bucketName))
+		requireContains(t, out, bucketName, "bucket.create output")
+
+		out = execShell(t, weedCmd, master, filer, "s3.bucket.list")
+		requireContains(t, out, bucketName, "bucket.list contains created")
+
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.delete -name %s", bucketName))
+
+		out = execShell(t, weedCmd, master, filer, "s3.bucket.list")
+		requireNotContains(t, out, bucketName, "bucket.list after delete")
+	})
+
+	t.Run("Owner", func(t *testing.T) {
+		bucketName := uniqueName("bkt-own")
+		ownerName := uniqueName("owner")
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.create -name %s", ownerName))
+		defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.delete -name %s", ownerName))
+
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.create -name %s", bucketName))
+		defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.delete -name %s", bucketName))
+
+		// Initially no owner.
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.owner -name %s", bucketName))
+		requireContains(t, out, "none", "initial owner none")
+
+		// Set owner.
+		execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.bucket.owner -name %s -owner %s", bucketName, ownerName))
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.owner -name %s", bucketName))
+		requireContains(t, out, ownerName, "owner set")
+
+		// Remove owner.
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.owner -name %s -delete", bucketName))
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.owner -name %s", bucketName))
+		requireContains(t, out, "none", "owner removed")
+	})
+
+	t.Run("Quota", func(t *testing.T) {
+		bucketName := uniqueName("bkt-quota")
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.create -name %s", bucketName))
+		defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.delete -name %s", bucketName))
+
+		execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.bucket.quota -name %s -op=set -sizeMB=1024", bucketName))
+
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.quota -name %s -op=get", bucketName))
+		requireContains(t, out, "1024", "quota.get shows size")
+
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.quota -name %s -op=disable", bucketName))
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.quota -name %s -op=enable", bucketName))
+
+		// Enforce should run on an empty bucket without error.
+		execShell(t, weedCmd, master, filer, "s3.bucket.quota.enforce -apply")
+
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.quota -name %s -op=remove", bucketName))
+	})
+
+	t.Run("Versioning", func(t *testing.T) {
+		bucketName := uniqueName("bkt-ver")
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.create -name %s", bucketName))
+		defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.delete -name %s", bucketName))
+
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.versioning -name %s -enable", bucketName))
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.versioning -name %s", bucketName))
+		requireContains(t, out, "Enabled", "versioning enabled")
+
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.versioning -name %s -suspend", bucketName))
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.versioning -name %s", bucketName))
+		requireContains(t, out, "Suspended", "versioning suspended")
+	})
+
+	t.Run("Lock", func(t *testing.T) {
+		bucketName := uniqueName("bkt-lock")
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.create -name %s", bucketName))
+		defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.delete -name %s", bucketName))
+
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.lock -name %s", bucketName))
+		requireContains(t, out, "Disabled", "lock initially disabled")
+
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.lock -name %s -enable", bucketName))
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.lock -name %s", bucketName))
+		requireContains(t, out, "Enabled", "lock enabled")
+
+		// Versioning should have been auto-enabled.
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.versioning -name %s", bucketName))
+		requireContains(t, out, "Enabled", "versioning auto-enabled by lock")
+	})
+
+	t.Run("CreateWithLock", func(t *testing.T) {
+		bucketName := uniqueName("bkt-wlock")
+		out := execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.bucket.create -name %s -withLock", bucketName))
+		// Cleanup may fail if the bucket contains locked objects; we created none.
+		defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.delete -name %s", bucketName))
+
+		requireContains(t, out, "Object Lock", "create -withLock output")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.lock -name %s", bucketName))
+		requireContains(t, out, "Enabled", "lock enabled after create -withLock")
+	})
+}
@@ -0,0 +1,86 @@
+package policy
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/require"
+)
+
+// TestShellConfigShow verifies s3.config.show outputs a summary of IAM config.
+func TestShellConfigShow(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	const weedCmd = "weed"
+	master := string(pb.NewServerAddress("127.0.0.1", cluster.masterPort, cluster.masterGrpcPort))
+	filer := string(pb.NewServerAddress("127.0.0.1", cluster.filerPort, cluster.filerGrpcPort))
+
+	userName := uniqueName("cfg-user")
+	groupName := uniqueName("cfg-grp")
+	execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.create -name %s", userName))
+	defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.delete -name %s", userName))
+	execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.group.create -name %s", groupName))
+	defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.group.delete -name %s", groupName))
+
+	out := execShell(t, weedCmd, master, filer, "s3.config.show")
+	requireContains(t, out, "S3 IAM Configuration Summary", "config.show header")
+	requireContains(t, out, userName, "config.show contains user")
+	requireContains(t, out, groupName, "config.show contains group")
+}
+
+// TestShellIAMExportImport does a roundtrip: create resources, export, delete, import, verify.
+func TestShellIAMExportImport(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	const weedCmd = "weed"
+	master := string(pb.NewServerAddress("127.0.0.1", cluster.masterPort, cluster.masterGrpcPort))
+	filer := string(pb.NewServerAddress("127.0.0.1", cluster.filerPort, cluster.filerGrpcPort))
+
+	userName := uniqueName("exp-user")
+	groupName := uniqueName("exp-grp")
+	execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.create -name %s", userName))
+	execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.group.create -name %s", groupName))
+
+	exportFile := filepath.Join(t.TempDir(), "iam_export.txt")
+	execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.iam.export -file %s", exportFile))
+
+	data, err := os.ReadFile(exportFile)
+	require.NoError(t, err)
+	content := string(data)
+	requireContains(t, content, userName, "export file contains user")
+	requireContains(t, content, groupName, "export file contains group")
+
+	// Delete the resources.
+	execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.delete -name %s", userName))
+	execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.group.delete -name %s", groupName))
+
+	out := execShell(t, weedCmd, master, filer, "s3.user.list")
+	requireNotContains(t, out, userName, "user gone before import")
+
+	// Import.
+	out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.iam.import -file %s -apply", exportFile))
+	requireContains(t, out, "Imported IAM configuration", "import output")
+
+	// Verify resources restored.
+	out = execShell(t, weedCmd, master, filer, "s3.user.list")
+	requireContains(t, out, userName, "user restored after import")
+	out = execShell(t, weedCmd, master, filer, "s3.group.list")
+	requireContains(t, out, groupName, "group restored after import")
+
+	// Cleanup.
+	execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.delete -name %s", userName))
+	execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.group.delete -name %s", groupName))
+}
@@ -0,0 +1,62 @@
+package policy
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/require"
+)
+
+// TestShellGroupLifecycle exercises s3.group.* commands end-to-end.
+func TestShellGroupLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	const weedCmd = "weed"
+	master := string(pb.NewServerAddress("127.0.0.1", cluster.masterPort, cluster.masterGrpcPort))
+	filer := string(pb.NewServerAddress("127.0.0.1", cluster.filerPort, cluster.filerGrpcPort))
+
+	groupName := uniqueName("grp")
+	userName := uniqueName("grpuser")
+
+	execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.create -name %s", userName))
+	defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.delete -name %s", userName))
+
+	t.Run("CreateShowList", func(t *testing.T) {
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.group.create -name %s", groupName))
+		requireContains(t, out, groupName, "group.create output")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.group.show -name %s", groupName))
+		requireContains(t, out, groupName, "group.show output")
+
+		out = execShell(t, weedCmd, master, filer, "s3.group.list")
+		requireContains(t, out, groupName, "group.list")
+	})
+
+	t.Run("AddRemoveUser", func(t *testing.T) {
+		out := execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.group.add-user -group %s -user %s", groupName, userName))
+		requireContains(t, out, userName, "group.add-user output")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.group.show -name %s", groupName))
+		requireContains(t, out, userName, "group.show after add")
+
+		out = execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.group.remove-user -group %s -user %s", groupName, userName))
+		requireContains(t, out, userName, "group.remove-user output")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.group.show -name %s", groupName))
+		requireNotContains(t, out, fmt.Sprintf("\"%s\"", userName), "group.show after remove")
+	})
+
+	t.Run("Delete", func(t *testing.T) {
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.group.delete -name %s", groupName))
+		out := execShell(t, weedCmd, master, filer, "s3.group.list")
+		requireNotContains(t, out, fmt.Sprintf("\"%s\"", groupName), "group.list after delete")
+	})
+}
@@ -0,0 +1,73 @@
+package policy
+
+import (
+	"strings"
+	"testing"
+)
+
+// requireContains fails the test if substr is not found in output.
+func requireContains(t *testing.T, output, substr, context string) {
+	t.Helper()
+	if !strings.Contains(output, substr) {
+		t.Fatalf("%s: expected output to contain %q\n--- output ---\n%s\n--- end ---", context, substr, output)
+	}
+}
+
+// requireNotContains fails the test if substr IS found in output.
+func requireNotContains(t *testing.T, output, substr, context string) {
+	t.Helper()
+	if strings.Contains(output, substr) {
+		t.Fatalf("%s: expected output to NOT contain %q\n--- output ---\n%s\n--- end ---", context, substr, output)
+	}
+}
+
+// extractFieldAfter returns the first occurrence of the value after a "Prefix: " line.
+// Example: extractFieldAfter(out, "Access Key:") -> "AKIAXXXX..."
+// Returns "" if not found.
+func extractFieldAfter(output, prefix string) string {
+	for _, line := range strings.Split(output, "\n") {
+		line = strings.TrimSpace(line)
+		if strings.HasPrefix(line, prefix) {
+			return strings.TrimSpace(strings.TrimPrefix(line, prefix))
+		}
+	}
+	return ""
+}
+
+// splitLines splits output into trimmed non-empty lines.
+func splitLines(output string) []string {
+	var lines []string
+	for _, line := range strings.Split(output, "\n") {
+		if trimmed := strings.TrimSpace(line); trimmed != "" {
+			lines = append(lines, trimmed)
+		}
+	}
+	return lines
+}
+
+// fieldsOf splits a line on whitespace.
+func fieldsOf(line string) []string {
+	return strings.Fields(line)
+}
+
+// extractServiceAccountID parses the tab-separated output of `s3.serviceaccount.list`
+// and returns the ID of the first row whose PARENT column matches parentUser.
+// The list output format is:
+//
+//	ID                                         PARENT     STATUS    DESCRIPTION
+//	sa:user-yyy:a1b2c3d4e5f6...                user-yyy   enabled   some desc
+func extractServiceAccountID(t *testing.T, listOutput, parentUser string) string {
+	t.Helper()
+	for _, line := range strings.Split(listOutput, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "ID") || strings.HasPrefix(line, "No service accounts") {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) >= 2 && fields[1] == parentUser {
+			return fields[0]
+		}
+	}
+	t.Fatalf("could not find service account with parent=%q in output:\n%s", parentUser, listOutput)
+	return ""
+}
@@ -0,0 +1,67 @@
+package policy
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/require"
+)
+
+// TestShellPolicyAttachDetach exercises s3.policy.attach and s3.policy.detach.
+func TestShellPolicyAttachDetach(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	const weedCmd = "weed"
+	master := string(pb.NewServerAddress("127.0.0.1", cluster.masterPort, cluster.masterGrpcPort))
+	filer := string(pb.NewServerAddress("127.0.0.1", cluster.filerPort, cluster.filerGrpcPort))
+
+	// Create a policy via file.
+	policyJSON := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`
+	tmpFile, err := os.CreateTemp("", "test_policy_*.json")
+	require.NoError(t, err)
+	defer os.Remove(tmpFile.Name())
+	_, err = tmpFile.WriteString(policyJSON)
+	require.NoError(t, err)
+	require.NoError(t, tmpFile.Close())
+
+	policyName := uniqueName("attach-pol")
+	userName := uniqueName("attach-user")
+
+	execShell(t, weedCmd, master, filer,
+		fmt.Sprintf("s3.policy -put -name=%s -file=%s", policyName, tmpFile.Name()))
+	defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.policy -delete -name=%s", policyName))
+
+	execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.create -name %s", userName))
+	defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.delete -name %s", userName))
+
+	t.Run("AttachAndVerify", func(t *testing.T) {
+		out := execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.policy.attach -policy %s -user %s", policyName, userName))
+		requireContains(t, out, policyName, "policy.attach output")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.show -name %s", userName))
+		requireContains(t, out, policyName, "user.show after attach")
+	})
+
+	t.Run("AttachIdempotent", func(t *testing.T) {
+		// Should succeed without error per the command's idempotent design.
+		execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.policy.attach -policy %s -user %s", policyName, userName))
+	})
+
+	t.Run("DetachAndVerify", func(t *testing.T) {
+		out := execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.policy.detach -policy %s -user %s", policyName, userName))
+		requireContains(t, out, policyName, "policy.detach output")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.show -name %s", userName))
+		requireNotContains(t, out, fmt.Sprintf("\"%s\"", policyName), "user.show after detach")
+	})
+}
@@ -0,0 +1,80 @@
+package policy
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/require"
+)
+
+// TestShellServiceAccountLifecycle exercises s3.serviceaccount.* commands end-to-end.
+func TestShellServiceAccountLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	const weedCmd = "weed"
+	master := string(pb.NewServerAddress("127.0.0.1", cluster.masterPort, cluster.masterGrpcPort))
+	filer := string(pb.NewServerAddress("127.0.0.1", cluster.filerPort, cluster.filerGrpcPort))
+
+	userName := uniqueName("sauser")
+	execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.create -name %s", userName))
+	defer execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.delete -name %s", userName))
+
+	var saID string
+
+	t.Run("CreateAndList", func(t *testing.T) {
+		description := "integration-test-sa"
+		out := execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.serviceaccount.create -user %s -description %s", userName, description))
+		requireContains(t, out, "Created service account", "serviceaccount.create")
+		requireContains(t, out, "Access Key:", "serviceaccount.create credentials")
+		requireContains(t, out, "Secret Key:", "serviceaccount.create credentials")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.serviceaccount.list -user %s", userName))
+		requireContains(t, out, userName, "serviceaccount.list parent column")
+		saID = extractServiceAccountID(t, out, userName)
+	})
+
+	t.Run("Show", func(t *testing.T) {
+		if saID == "" {
+			t.Skip("no saID extracted")
+		}
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.serviceaccount.show -id %s", saID))
+		requireContains(t, out, saID, "show contains id")
+		requireContains(t, out, userName, "show contains parent")
+		requireContains(t, out, "enabled", "show contains status")
+	})
+
+	t.Run("CreateWithActions", func(t *testing.T) {
+		out := execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.serviceaccount.create -user %s -actions Read,List -expiry 24h", userName))
+		requireContains(t, out, "Access Key:", "serviceaccount.create with options")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.serviceaccount.list -user %s", userName))
+		// Should now have at least 2 service accounts — count lines that start with parent in field[1]
+		count := 0
+		for _, line := range splitLines(out) {
+			fields := fieldsOf(line)
+			if len(fields) >= 2 && fields[1] == userName {
+				count++
+			}
+		}
+		if count < 2 {
+			t.Fatalf("expected at least 2 service accounts for %s, got %d\n%s", userName, count, out)
+		}
+	})
+
+	t.Run("Delete", func(t *testing.T) {
+		if saID == "" {
+			t.Skip("no saID extracted")
+		}
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.serviceaccount.delete -id %s", saID))
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.serviceaccount.list -user %s", userName))
+		requireNotContains(t, out, saID, "deleted sa removed from list")
+	})
+}
@@ -0,0 +1,110 @@
+package policy
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/require"
+)
+
+// TestShellUserLifecycle exercises the s3.user.* commands end-to-end:
+// create, show, list, enable, disable, delete, and provision.
+func TestShellUserLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	const weedCmd = "weed"
+	master := string(pb.NewServerAddress("127.0.0.1", cluster.masterPort, cluster.masterGrpcPort))
+	filer := string(pb.NewServerAddress("127.0.0.1", cluster.filerPort, cluster.filerGrpcPort))
+
+	t.Run("CreateShowListDelete", func(t *testing.T) {
+		userName := uniqueName("user")
+
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.create -name %s", userName))
+		requireContains(t, out, userName, "user.create output")
+		requireContains(t, out, "access_key", "user.create JSON")
+		requireContains(t, out, "Secret Key:", "user.create stderr")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.show -name %s", userName))
+		requireContains(t, out, userName, "user.show")
+
+		out = execShell(t, weedCmd, master, filer, "s3.user.list")
+		requireContains(t, out, userName, "user.list")
+
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.delete -name %s", userName))
+
+		out = execShell(t, weedCmd, master, filer, "s3.user.list")
+		requireNotContains(t, out, userName, "user.list after delete")
+	})
+
+	t.Run("CreateWithExplicitKeys", func(t *testing.T) {
+		userName := uniqueName("user-expl")
+		ak := "TESTAK1234567890ABCD"
+		sk := "testsecretkey1234567890abcdefghijklmnopq"
+
+		out := execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.user.create -name %s -access_key %s -secret_key %s", userName, ak, sk))
+		requireContains(t, out, ak, "user.create with explicit keys")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.show -name %s", userName))
+		requireContains(t, out, ak, "user.show reveals access key")
+
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.delete -name %s", userName))
+	})
+
+	t.Run("EnableDisable", func(t *testing.T) {
+		userName := uniqueName("user-toggle")
+
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.create -name %s", userName))
+
+		out := execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.disable -name %s", userName))
+		requireContains(t, out, "disabled", "user.disable")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.show -name %s", userName))
+		requireContains(t, out, "disabled", "user.show after disable")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.enable -name %s", userName))
+		requireContains(t, out, "enabled", "user.enable")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.show -name %s", userName))
+		requireContains(t, out, "enabled", "user.show after enable")
+
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.delete -name %s", userName))
+	})
+
+	t.Run("Provision", func(t *testing.T) {
+		userName := uniqueName("prov-user")
+		bucketName := uniqueName("prov-bkt")
+
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.create -name %s", bucketName))
+
+		out := execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.user.provision -name %s -bucket %s -role readwrite", userName, bucketName))
+		requireContains(t, out, "Created policy", "provision output")
+		requireContains(t, out, "Created user", "provision output")
+		requireContains(t, out, "Access Key:", "provision credentials")
+		requireContains(t, out, "Secret Key:", "provision credentials")
+
+		out = execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.show -name %s", userName))
+		requireContains(t, out, userName, "user.show after provision")
+
+		// Second call with same user but different bucket/role should succeed without creating duplicate user.
+		bucket2 := uniqueName("prov-bkt2")
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.create -name %s", bucket2))
+		out = execShell(t, weedCmd, master, filer,
+			fmt.Sprintf("s3.user.provision -name %s -bucket %s -role readonly", userName, bucket2))
+		requireContains(t, out, "already exists", "provision on existing user")
+		requireContains(t, out, "Created policy", "second policy created")
+		requireContains(t, out, "Attached policy", "second policy attached to existing user")
+		requireNotContains(t, out, "Access Key:", "no new credentials for existing user")
+
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.user.delete -name %s", userName))
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.delete -name %s", bucketName))
+		execShell(t, weedCmd, master, filer, fmt.Sprintf("s3.bucket.delete -name %s", bucket2))
+	})
+}
@@ -70,7 +70,8 @@ print("WRITE_COUNT=" + str(count))
 		"issue-8285/output/_temporary/0/",
 		"issue-8285/output/_temporary/0/_temporary/",
 	}
-	lingering := waitForObjectsToDisappear(t, env, "test", temporaryCandidates, 35*time.Second)
+	// Empty folder cleanup has a 2m default delay + 30s processor interval
+	lingering := waitForObjectsToDisappear(t, env, "test", temporaryCandidates, 3*time.Minute)
 	if len(lingering) > 0 {
 		t.Fatalf("issue #8285 regression detected: lingering temporary directories: %v", lingering)
 	}
@@ -14,7 +14,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/testcontainers/testcontainers-go"
 )

@@ -94,9 +94,10 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}

-	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
-	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
-	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
+	ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3"})
+	env.masterPort = ports[0]
+	env.filerPort = ports[1]
+	env.s3Port = ports[2]

 	bindIP := testutil.FindBindIP()
 	iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)
@@ -0,0 +1,465 @@
+package catalog
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+)
+
+// oauthTestEnv holds a weed mini instance started with IAM credentials
+// so that the OAuth token endpoint is functional.
+type oauthTestEnv struct {
+	seaweedDir     string
+	weedBinary     string
+	dataDir        string
+	bindIP         string
+	s3Port         int
+	s3GrpcPort     int
+	icebergPort    int
+	masterPort     int
+	masterGrpcPort int
+	filerPort      int
+	filerGrpcPort  int
+	volumePort     int
+	volumeGrpcPort int
+	webdavPort     int
+	weedProcess    *exec.Cmd
+	weedCancel     context.CancelFunc
+	accessKey      string
+	secretKey      string
+}
+
+func newOAuthTestEnv(t *testing.T) *oauthTestEnv {
+	t.Helper()
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("get working directory: %v", err)
+	}
+
+	seaweedDir := wd
+	for i := 0; i < 6; i++ {
+		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
+			break
+		}
+		seaweedDir = filepath.Dir(seaweedDir)
+	}
+
+	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
+	if info, err := os.Stat(weedBinary); err != nil || info.IsDir() {
+		weedBinary = "weed"
+		if _, err := exec.LookPath(weedBinary); err != nil {
+			t.Skip("weed binary not found, skipping integration test")
+		}
+	}
+
+	dataDir, err := os.MkdirTemp("", "seaweed-oauth-test-*")
+	if err != nil {
+		t.Fatalf("create temp dir: %v", err)
+	}
+
+	bindIP := testutil.FindBindIP()
+	ports := testutil.MustAllocatePorts(t, 10)
+
+	return &oauthTestEnv{
+		seaweedDir:     seaweedDir,
+		weedBinary:     weedBinary,
+		dataDir:        dataDir,
+		bindIP:         bindIP,
+		masterPort:     ports[0],
+		masterGrpcPort: ports[1],
+		volumePort:     ports[2],
+		volumeGrpcPort: ports[3],
+		filerPort:      ports[4],
+		filerGrpcPort:  ports[5],
+		s3Port:         ports[6],
+		s3GrpcPort:     ports[7],
+		icebergPort:    ports[8],
+		webdavPort:     ports[9],
+		accessKey:      "AKIAIOSFODNN7EXAMPLE",
+		secretKey:      "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+	}
+}
+
+func (env *oauthTestEnv) start(t *testing.T) {
+	t.Helper()
+
+	iamConfigPath, err := testutil.WriteIAMConfig(env.dataDir, env.accessKey, env.secretKey)
+	if err != nil {
+		t.Fatalf("write IAM config: %v", err)
+	}
+
+	securityToml := filepath.Join(env.dataDir, "security.toml")
+	if err := os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644); err != nil {
+		t.Fatalf("write security.toml: %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	env.weedCancel = cancel
+
+	cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
+		"-master.port", fmt.Sprintf("%d", env.masterPort),
+		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
+		"-volume.port", fmt.Sprintf("%d", env.volumePort),
+		"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
+		"-filer.port", fmt.Sprintf("%d", env.filerPort),
+		"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
+		"-s3.port", fmt.Sprintf("%d", env.s3Port),
+		"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
+		"-s3.port.iceberg", fmt.Sprintf("%d", env.icebergPort),
+		"-webdav.port", fmt.Sprintf("%d", env.webdavPort),
+		"-s3.config", iamConfigPath,
+		"-ip", env.bindIP,
+		"-ip.bind", "0.0.0.0",
+		"-dir", env.dataDir,
+	)
+	cmd.Dir = env.dataDir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Env = append(os.Environ(),
+		"AWS_ACCESS_KEY_ID="+env.accessKey,
+		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
+	)
+
+	if err := cmd.Start(); err != nil {
+		cancel()
+		t.Fatalf("start weed mini: %v", err)
+	}
+	env.weedProcess = cmd
+
+	icebergURL := fmt.Sprintf("http://%s:%d/v1/config", env.bindIP, env.icebergPort)
+	if !testutil.WaitForService(icebergURL, 30*time.Second) {
+		cancel()
+		cmd.Wait()
+		t.Fatalf("Iceberg REST API did not become ready at %s", icebergURL)
+	}
+}
+
+func (env *oauthTestEnv) cleanup(t *testing.T) {
+	t.Helper()
+	if env.weedCancel != nil {
+		env.weedCancel()
+	}
+	if env.weedProcess != nil {
+		env.weedProcess.Wait()
+	}
+	if env.dataDir != "" {
+		os.RemoveAll(env.dataDir)
+	}
+}
+
+func (env *oauthTestEnv) icebergURL() string {
+	return fmt.Sprintf("http://%s:%d", env.bindIP, env.icebergPort)
+}
+
+// TestOAuthTokenEndpoint tests the /v1/oauth/tokens endpoint directly via HTTP.
+func TestOAuthTokenEndpoint(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := newOAuthTestEnv(t)
+	defer env.cleanup(t)
+	env.start(t)
+
+	t.Run("valid credentials", func(t *testing.T) {
+		token := requestOAuthToken(t, env, env.accessKey, env.secretKey)
+		if token == "" {
+			t.Fatal("expected non-empty token")
+		}
+	})
+
+	t.Run("invalid secret", func(t *testing.T) {
+		resp, err := http.PostForm(env.icebergURL()+"/v1/oauth/tokens", url.Values{
+			"grant_type":    {"client_credentials"},
+			"client_id":     {env.accessKey},
+			"client_secret": {"wrong-secret"},
+		})
+		if err != nil {
+			t.Fatalf("POST /v1/oauth/tokens: %v", err)
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusUnauthorized {
+			body, _ := io.ReadAll(resp.Body)
+			t.Fatalf("expected 401, got %d: %s", resp.StatusCode, body)
+		}
+	})
+
+	t.Run("missing grant_type", func(t *testing.T) {
+		resp, err := http.PostForm(env.icebergURL()+"/v1/oauth/tokens", url.Values{
+			"client_id":     {env.accessKey},
+			"client_secret": {env.secretKey},
+		})
+		if err != nil {
+			t.Fatalf("POST /v1/oauth/tokens: %v", err)
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusBadRequest {
+			body, _ := io.ReadAll(resp.Body)
+			t.Fatalf("expected 400, got %d: %s", resp.StatusCode, body)
+		}
+	})
+
+	t.Run("bearer token auth on catalog endpoint", func(t *testing.T) {
+		token := requestOAuthToken(t, env, env.accessKey, env.secretKey)
+
+		// Use the token to call the catalog
+		req, err := http.NewRequest(http.MethodGet, env.icebergURL()+"/v1/namespaces", nil)
+		if err != nil {
+			t.Fatalf("create request: %v", err)
+		}
+		req.Header.Set("Authorization", "Bearer "+token)
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatalf("GET /v1/namespaces with Bearer: %v", err)
+		}
+		defer resp.Body.Close()
+
+		// Auth should pass. We accept 200 (success) or 500 (missing warehouse bucket
+		// is an internal error, not an auth error). Reject 401/403/404/405.
+		body, _ := io.ReadAll(resp.Body)
+		switch resp.StatusCode {
+		case http.StatusOK, http.StatusInternalServerError:
+			t.Logf("Bearer token auth succeeded, status=%d", resp.StatusCode)
+		default:
+			t.Fatalf("Bearer auth failed unexpectedly: status=%d body=%s", resp.StatusCode, body)
+		}
+	})
+}
+
+// TestDuckDBOAuthIntegration tests that DuckDB can connect to the Iceberg REST
+// catalog using the OAuth2 client_credentials flow (CREATE SECRET with client_id
+// and client_secret). This is the scenario reported in issue #9015.
+func TestDuckDBOAuthIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	if !testutil.HasDocker() {
+		t.Skip("Docker not available, skipping DuckDB OAuth integration test")
+	}
+
+	env := newOAuthTestEnv(t)
+	defer env.cleanup(t)
+	env.start(t)
+
+	// Create a table bucket and namespace so DuckDB has something to query
+	bucketName := "duckdb-oauth-" + randomSuffix()
+	createTableBucketViaShell(t, env, bucketName)
+
+	// Create a namespace via the Iceberg REST API using OAuth token
+	token := requestOAuthToken(t, env, env.accessKey, env.secretKey)
+	createNamespaceWithToken(t, env, token, bucketName, "testns")
+
+	sqlContent := fmt.Sprintf(`
+INSTALL iceberg;
+LOAD iceberg;
+
+CREATE SECRET iceberg_secret (
+    TYPE ICEBERG,
+    ENDPOINT 'http://host.docker.internal:%d',
+    CLIENT_ID '%s',
+    CLIENT_SECRET '%s',
+    SCOPE 's3://%s/'
+);
+
+CREATE SECRET s3_secret (
+    TYPE S3,
+    KEY_ID '%s',
+    SECRET '%s',
+    ENDPOINT 'host.docker.internal:%d',
+    URL_STYLE 'path',
+    USE_SSL false,
+    SCOPE 's3://%s/'
+);
+
+SELECT 'OAuth token obtained successfully' as status;
+
+-- Try listing namespaces via the Iceberg catalog
+SELECT * FROM iceberg_scan('iceberg_secret', ALLOW_MOVED_PATHS => TRUE) LIMIT 0;
+`, env.icebergPort, env.accessKey, env.secretKey, bucketName,
+		env.accessKey, env.secretKey, env.s3Port, bucketName)
+
+	sqlFile := filepath.Join(env.dataDir, "duckdb_oauth_test.sql")
+	if err := os.WriteFile(sqlFile, []byte(sqlContent), 0644); err != nil {
+		t.Fatalf("write SQL file: %v", err)
+	}
+
+	// Run DuckDB in Docker.
+	// We use a simple test: get a token, create a secret, and verify the
+	// Iceberg extension can communicate with the catalog.
+	// The simpler fallback test just verifies CREATE SECRET succeeds (no 404).
+	fallbackSQL := fmt.Sprintf(`
+INSTALL iceberg;
+LOAD iceberg;
+
+CREATE SECRET (
+    TYPE ICEBERG,
+    ENDPOINT 'http://host.docker.internal:%d',
+    CLIENT_ID '%s',
+    CLIENT_SECRET '%s'
+);
+
+SELECT 'DuckDB Iceberg OAuth secret created successfully' as result;
+`, env.icebergPort, env.accessKey, env.secretKey)
+
+	fallbackFile := filepath.Join(env.dataDir, "duckdb_oauth_fallback.sql")
+	if err := os.WriteFile(fallbackFile, []byte(fallbackSQL), 0644); err != nil {
+		t.Fatalf("write fallback SQL file: %v", err)
+	}
+
+	// Try the full SQL script first (CREATE SECRET + catalog access).
+	// Fall back to the simple CREATE SECRET test if iceberg_scan isn't supported.
+	cmd := exec.Command("docker", "run", "--rm",
+		"-v", fmt.Sprintf("%s:/test", env.dataDir),
+		"--add-host", "host.docker.internal:host-gateway",
+		"--entrypoint", "duckdb",
+		"duckdb/duckdb:latest",
+		"-init", "/test/duckdb_oauth_test.sql",
+		"-c", "SELECT 1",
+	)
+
+	output, err := cmd.CombinedOutput()
+	outputStr := string(output)
+	t.Logf("DuckDB output:\n%s", outputStr)
+
+	if err != nil {
+		if strings.Contains(outputStr, "iceberg extension is not available") ||
+			strings.Contains(outputStr, "Failed to load") {
+			t.Skip("Skipping: Iceberg extension not available in DuckDB Docker image")
+		}
+		// The key check: the old error was "HTTP NotFound_404" on /v1/oauth/tokens.
+		// With our fix, this should no longer happen.
+		if strings.Contains(outputStr, "NotFound_404") && strings.Contains(outputStr, "/v1/oauth/tokens") {
+			t.Fatal("OAuth token endpoint returned 404 - the fix is not working")
+		}
+		// If iceberg_scan failed but CREATE SECRET worked, fall back to simpler test
+		if strings.Contains(outputStr, "OAuth token obtained successfully") {
+			t.Logf("Full SQL had partial success (token obtained), iceberg_scan may not be supported. Continuing.")
+		} else {
+			// Try the fallback script that only tests CREATE SECRET
+			t.Logf("Full SQL failed, trying fallback CREATE SECRET test...")
+			fallbackCmd := exec.Command("docker", "run", "--rm",
+				"-v", fmt.Sprintf("%s:/test", env.dataDir),
+				"--add-host", "host.docker.internal:host-gateway",
+				"--entrypoint", "duckdb",
+				"duckdb/duckdb:latest",
+				"-init", "/test/duckdb_oauth_fallback.sql",
+				"-c", "SELECT 1",
+			)
+			fallbackOutput, fallbackErr := fallbackCmd.CombinedOutput()
+			fallbackStr := string(fallbackOutput)
+			t.Logf("DuckDB fallback output:\n%s", fallbackStr)
+
+			if fallbackErr != nil {
+				if strings.Contains(fallbackStr, "NotFound_404") && strings.Contains(fallbackStr, "/v1/oauth/tokens") {
+					t.Fatal("OAuth token endpoint returned 404 - the fix is not working")
+				}
+				t.Fatalf("DuckDB fallback also failed: %v\nOutput: %s", fallbackErr, fallbackStr)
+			}
+			if !strings.Contains(fallbackStr, "DuckDB Iceberg OAuth secret created successfully") {
+				t.Errorf("expected success message in fallback output, got:\n%s", fallbackStr)
+			}
+			return
+		}
+	}
+
+	if strings.Contains(outputStr, "OAuth token obtained successfully") {
+		t.Logf("DuckDB OAuth CREATE SECRET succeeded")
+	} else {
+		t.Errorf("expected OAuth success message in output, got:\n%s", outputStr)
+	}
+}
+
+// requestOAuthToken obtains a bearer token from the OAuth endpoint.
+func requestOAuthToken(t *testing.T, env *oauthTestEnv, accessKey, secretKey string) string {
+	t.Helper()
+
+	resp, err := http.PostForm(env.icebergURL()+"/v1/oauth/tokens", url.Values{
+		"grant_type":    {"client_credentials"},
+		"client_id":     {accessKey},
+		"client_secret": {secretKey},
+	})
+	if err != nil {
+		t.Fatalf("POST /v1/oauth/tokens: %v", err)
+	}
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("OAuth token request failed: status=%d body=%s", resp.StatusCode, body)
+	}
+
+	var tokenResp struct {
+		AccessToken string `json:"access_token"`
+		TokenType   string `json:"token_type"`
+		ExpiresIn   int    `json:"expires_in"`
+	}
+	if err := json.Unmarshal(body, &tokenResp); err != nil {
+		t.Fatalf("decode token response: %v", err)
+	}
+
+	if tokenResp.AccessToken == "" {
+		t.Fatal("got empty access_token")
+	}
+	if tokenResp.TokenType != "bearer" {
+		t.Errorf("expected token_type=bearer, got %s", tokenResp.TokenType)
+	}
+	return tokenResp.AccessToken
+}
+
+// createTableBucketViaShell creates a table bucket using weed shell,
+// which bypasses S3 auth. This is the same approach used by the Trino tests.
+func createTableBucketViaShell(t *testing.T, env *oauthTestEnv, bucketName string) {
+	t.Helper()
+
+	cmd := exec.Command(env.weedBinary, "shell",
+		fmt.Sprintf("-master=%s:%d.%d", env.bindIP, env.masterPort, env.masterGrpcPort),
+	)
+	cmd.Stdin = strings.NewReader(fmt.Sprintf("s3tables.bucket -create -name %s -account 000000000000\nexit\n", bucketName))
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("create table bucket %s via weed shell: %v\nOutput: %s", bucketName, err, output)
+	}
+	t.Logf("Created table bucket %s", bucketName)
+}
+
+// createNamespaceWithToken creates a namespace using a Bearer token.
+func createNamespaceWithToken(t *testing.T, env *oauthTestEnv, token, bucketName, namespace string) {
+	t.Helper()
+
+	path := fmt.Sprintf("/v1/%s/namespaces", bucketName)
+	body := fmt.Sprintf(`{"namespace":["%s"]}`, namespace)
+	req, err := http.NewRequest(http.MethodPost, env.icebergURL()+path, strings.NewReader(body))
+	if err != nil {
+		t.Fatalf("create request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "Bearer "+token)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("create namespace: %v", err)
+	}
+	defer resp.Body.Close()
+
+	respBody, _ := io.ReadAll(resp.Body)
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusConflict {
+		t.Fatalf("create namespace failed: status=%d body=%s", resp.StatusCode, respBody)
+	}
+	t.Logf("Created namespace %s in bucket %s", namespace, bucketName)
+}
@@ -9,7 +9,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -17,6 +16,8 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )

 // sharedEnv is the single TestEnvironment shared across all tests in this package.
@@ -69,23 +70,6 @@ type TestEnvironment struct {
 	dockerAvailable bool
 }

-// hasDocker checks if Docker is available
-func hasDocker() bool {
-	cmd := exec.Command("docker", "version")
-	return cmd.Run() == nil
-}
-
-// getFreePort returns an available ephemeral port and its listener
-func getFreePort() (int, net.Listener, error) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		return 0, nil, err
-	}
-
-	addr := listener.Addr().(*net.TCPAddr)
-	return addr.Port, listener, nil
-}
-
 // newTestEnvironmentForMain creates a TestEnvironment without calling t.Fatalf so it
 // can be used from TestMain (which has no *testing.T).
 func newTestEnvironmentForMain() (*TestEnvironment, error) {
@@ -118,95 +102,27 @@ func newTestEnvironmentForMain() (*TestEnvironment, error) {
 		return nil, fmt.Errorf("create temp dir: %w", err)
 	}

-	// Allocate free ephemeral ports for each service
-	var listeners []net.Listener
-	closeListeners := func() {
-		for _, l := range listeners {
-			l.Close()
-		}
-	}
-
-	var l net.Listener
-	s3Port, l, err := getFreePort()
+	// Allocate 9 unique ports atomically: s3, iceberg, s3Grpc, master, masterGrpc,
+	// filer, filerGrpc, volume, volumeGrpc
+	ports, err := testutil.AllocatePorts(9)
 	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for S3: %w", err)
+		return nil, fmt.Errorf("allocate ports: %w", err)
 	}
-	listeners = append(listeners, l)
-
-	icebergPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Iceberg: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	s3GrpcPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for S3 gRPC: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	masterPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Master: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	masterGrpcPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Master gRPC: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	filerPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Filer: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	filerGrpcPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Filer gRPC: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	volumePort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Volume: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	volumeGrpcPort, l, err := getFreePort()
-	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Volume gRPC: %w", err)
-	}
-	listeners = append(listeners, l)
-
-	// Release the port reservations so weed mini can bind to them
-	closeListeners()

 	return &TestEnvironment{
 		seaweedDir:      seaweedDir,
 		weedBinary:      weedBinary,
 		dataDir:         dataDir,
-		s3Port:          s3Port,
-		s3GrpcPort:      s3GrpcPort,
-		icebergPort:     icebergPort,
-		masterPort:      masterPort,
-		masterGrpcPort:  masterGrpcPort,
-		filerPort:       filerPort,
-		filerGrpcPort:   filerGrpcPort,
-		volumePort:      volumePort,
-		volumeGrpcPort:  volumeGrpcPort,
-		dockerAvailable: hasDocker(),
+		s3Port:          ports[0],
+		s3GrpcPort:      ports[1],
+		icebergPort:     ports[2],
+		masterPort:      ports[3],
+		masterGrpcPort:  ports[4],
+		filerPort:       ports[5],
+		filerGrpcPort:   ports[6],
+		volumePort:      ports[7],
+		volumeGrpcPort:  ports[8],
+		dockerAvailable: testutil.HasDocker(),
 	}, nil
 }

@@ -18,7 +18,7 @@ import (

 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )

 var (
@@ -114,11 +114,12 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}

-	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
-	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
-	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
-	env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")
-	env.risingwavePort = testutil.MustFreeMiniPort(t, "RisingWave")
+	ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg", "RisingWave"})
+	env.masterPort = ports[0]
+	env.filerPort = ports[1]
+	env.s3Port = ports[2]
+	env.icebergRestPort = ports[3]
+	env.risingwavePort = ports[4]

 	env.bindIP = testutil.FindBindIP()

@@ -16,7 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/testcontainers/testcontainers-go"
 )

@@ -90,10 +90,11 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		t.Fatalf("failed to create temp directory: %v", err)
 	}

-	env.masterPort = testutil.MustFreeMiniPort(t, "Master")
-	env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
-	env.s3Port = testutil.MustFreeMiniPort(t, "S3")
-	env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")
+	ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg"})
+	env.masterPort = ports[0]
+	env.filerPort = ports[1]
+	env.s3Port = ports[2]
+	env.icebergRestPort = ports[3]

 	bindIP := testutil.FindBindIP()

@@ -277,3 +277,80 @@ print(f"Count at snapshot: {count}")

 	t.Logf(">>> Time travel test passed")
 }
+
+// TestSparkMultiLevelNamespace tests that multi-level namespaces produce correct
+// S3 paths (dot-separated) so that Spark can read back the data it writes.
+// Regression test for https://github.com/seaweedfs/seaweedfs/issues/8959
+func TestSparkMultiLevelNamespace(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env, _, _ := setupSparkTestEnv(t)
+
+	// Use a two-level namespace like "analytics.daily"
+	nsLevel1 := "analytics_" + randomString(4)
+	nsLevel2 := "daily_" + randomString(4)
+	multiNs := fmt.Sprintf("%s.%s", nsLevel1, nsLevel2)
+	tableName := "events_" + randomString(4)
+
+	// Create multi-level namespace
+	t.Logf(">>> Creating multi-level namespace: %s", multiNs)
+	createNsSQL := fmt.Sprintf(`
+spark.sql("CREATE NAMESPACE iceberg.%s")
+print("Namespace created")
+`, multiNs)
+	output := runSparkPySQL(t, env.sparkContainer, createNsSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Namespace created") {
+		t.Fatalf("multi-level namespace creation failed, output: %s", output)
+	}
+
+	// Create table under multi-level namespace
+	t.Logf(">>> Creating table under multi-level namespace")
+	createTableSQL := fmt.Sprintf(`
+spark.sql("""
+CREATE TABLE iceberg.%s.%s (
+    id INT,
+    event STRING,
+    ts TIMESTAMP
+)
+USING iceberg
+""")
+print("Table created")
+`, multiNs, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, createTableSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Table created") {
+		t.Fatalf("table creation under multi-level namespace failed, output: %s", output)
+	}
+
+	// Insert data
+	t.Logf(">>> Inserting data into multi-level namespace table")
+	insertSQL := fmt.Sprintf(`
+spark.sql("""
+INSERT INTO iceberg.%s.%s VALUES
+    (1, 'click', TIMESTAMP '2025-01-01 00:00:00'),
+    (2, 'view',  TIMESTAMP '2025-01-01 01:00:00'),
+    (3, 'click', TIMESTAMP '2025-01-02 00:00:00')
+""")
+print("Data inserted")
+`, multiNs, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, insertSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Data inserted") {
+		t.Fatalf("data insertion failed, output: %s", output)
+	}
+
+	// Query data back — this is the key test: if the namespace path separator
+	// was wrong (\x1F instead of "."), Spark would not find the data files.
+	t.Logf(">>> Querying data from multi-level namespace table")
+	querySQL := fmt.Sprintf(`
+result = spark.sql("SELECT COUNT(*) as count FROM iceberg.%s.%s")
+count = result.collect()[0]['count']
+print(f"Row count: {count}")
+`, multiNs, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, querySQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Row count: 3") {
+		t.Errorf("expected row count 3 from multi-level namespace table, got output: %s", output)
+	}
+
+	t.Logf(">>> Multi-level namespace test passed")
+}
@@ -17,7 +17,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )

 type TestEnvironment struct {
@@ -40,7 +40,6 @@ type TestEnvironment struct {
 	dockerAvailable bool
 	accessKey       string
 	secretKey       string
-	closers         []io.Closer
 }

 func TestTrinoIcebergCatalog(t *testing.T) {
@@ -82,6 +81,101 @@ func TestTrinoIcebergCatalog(t *testing.T) {
 	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SHOW TABLES FROM iceberg.%s", schemaName))
 }

+// TestTrinoMultiLevelNamespace tests that multi-level namespaces (dot-separated)
+// produce correct S3 paths so Trino can read back data it writes.
+// Regression test for https://github.com/seaweedfs/seaweedfs/issues/8959
+func TestTrinoMultiLevelNamespace(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	if !env.dockerAvailable {
+		t.Skip("Docker not available, skipping Trino integration test")
+	}
+
+	t.Logf(">>> Starting SeaweedFS...")
+	env.StartSeaweedFS(t)
+
+	tableBucket := "iceberg-tables"
+	createTableBucket(t, env, tableBucket)
+
+	configDir := env.writeTrinoConfig(t, tableBucket, withNestedNamespace())
+	env.startTrinoContainer(t, configDir)
+	waitForTrino(t, env.trinoContainer, 60*time.Second)
+
+	// Use a two-level namespace: "analytics.daily"
+	nsLevel1 := "analytics_" + randomString(4)
+	nsLevel2 := "daily_" + randomString(4)
+	flatNs := fmt.Sprintf("%s.%s", nsLevel1, nsLevel2)
+	// Trino uses double-quoted schema names for multi-level namespaces
+	multiNs := fmt.Sprintf(`"%s"`, flatNs)
+	tableName := "events_" + randomString(4)
+
+	// Create multi-level namespace (schema)
+	t.Logf(">>> Creating multi-level schema: %s", flatNs)
+	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", multiNs))
+
+	// Verify the schema shows up
+	output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
+	if !strings.Contains(output, flatNs) {
+		t.Fatalf("Expected schema %s in output:\n%s", flatNs, output)
+	}
+
+	// Create table with explicit location to avoid non-empty location conflict.
+	// The location uses the dot-separated namespace — if #8959 regresses
+	// (unit separator instead of dot), data would be written to the wrong path.
+	tableLocation := fmt.Sprintf("s3://%s/%s/%s_%s", tableBucket, flatNs, tableName, randomString(6))
+	t.Logf(">>> Creating table at location: %s", tableLocation)
+	createSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS iceberg.%s.%s (
+		id INTEGER,
+		event VARCHAR,
+		ts TIMESTAMP(6)
+	) WITH (
+		format = 'PARQUET',
+		location = '%s'
+	)`, multiNs, tableName, tableLocation)
+	runTrinoSQLAllowExists(t, env.trinoContainer, createSQL)
+
+	// Insert data
+	t.Logf(">>> Inserting data into multi-level namespace table")
+	runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`
+		INSERT INTO iceberg.%s.%s VALUES
+			(1, 'click', TIMESTAMP '2025-01-01 00:00:00'),
+			(2, 'view',  TIMESTAMP '2025-01-01 01:00:00'),
+			(3, 'click', TIMESTAMP '2025-01-02 00:00:00')
+	`, multiNs, tableName))
+
+	// Query data back — if the namespace path separator were wrong (\x1F
+	// instead of "."), the metadata location would point to a non-existent
+	// S3 path and this query would fail.
+	t.Logf(">>> Querying data from multi-level namespace table")
+	countOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
+		"SELECT count(*) FROM iceberg.%s.%s", multiNs, tableName))
+	rowCount := mustParseCSVInt64(t, countOutput)
+	if rowCount != 3 {
+		t.Fatalf("expected row count 3, got %d", rowCount)
+	}
+
+	// Verify the S3 file path contains the dot-separated namespace, not \x1F.
+	filesOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
+		`SELECT file_path FROM iceberg.%s."%s$files" LIMIT 1`, multiNs, tableName))
+	filePath := strings.TrimSpace(filesOutput)
+	if filePath == "" {
+		t.Fatalf("expected at least one data file, got empty output")
+	}
+	if !strings.Contains(filePath, flatNs+"/") {
+		t.Errorf("expected file path to contain dot-separated namespace %q, got: %s", flatNs, filePath)
+	}
+	if strings.Contains(filePath, "\x1F") {
+		t.Errorf("file path contains unit separator (\\x1F), expected dot separator: %s", filePath)
+	}
+
+	t.Logf(">>> Trino multi-level namespace test passed")
+}
+
 func NewTestEnvironment(t *testing.T) *TestEnvironment {
 	t.Helper()

@@ -119,19 +213,24 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {

 	bindIP := testutil.FindBindIP()

-	env := &TestEnvironment{
-		seaweedDir: seaweedDir,
-		weedBinary: weedBinary,
-		dataDir:    dataDir,
-		bindIP:     bindIP,
-		closers:    []io.Closer{},
-	}
+	// 9 ports: master(2), volume(2), filer(2), s3(2), iceberg(1)
+	ports := testutil.MustAllocatePorts(t, 9)

-	env.masterPort, env.masterGrpcPort = env.mustFreePortPair("Master")
-	env.volumePort, env.volumeGrpcPort = env.mustFreePortPair("Volume")
-	env.filerPort, env.filerGrpcPort = env.mustFreePortPair("Filer")
-	env.s3Port, env.s3GrpcPort = env.mustFreePortPair("S3")
-	env.icebergPort = env.mustFreePort("Iceberg")
+	env := &TestEnvironment{
+		seaweedDir:     seaweedDir,
+		weedBinary:     weedBinary,
+		dataDir:        dataDir,
+		bindIP:         bindIP,
+		masterPort:     ports[0],
+		masterGrpcPort: ports[1],
+		volumePort:     ports[2],
+		volumeGrpcPort: ports[3],
+		filerPort:      ports[4],
+		filerGrpcPort:  ports[5],
+		s3Port:         ports[6],
+		s3GrpcPort:     ports[7],
+		icebergPort:    ports[8],
+	}

 	env.dockerAvailable = hasDocker()
 	env.accessKey = "AKIAIOSFODNN7EXAMPLE"
@@ -140,47 +239,6 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 	return env
 }

-func (env *TestEnvironment) mustFreePort(name string) int {
-	port, closer, err := getFreePort()
-	if err != nil {
-		panic(fmt.Sprintf("Failed to get free port for %s: %v", name, err))
-	}
-	env.closers = append(env.closers, closer)
-	return port
-}
-
-func (env *TestEnvironment) mustFreePortPair(name string) (int, int) {
-	httpPort, httpCloser, grpcPort, grpcCloser, err := findAvailablePortPair()
-	if err != nil {
-		panic(fmt.Sprintf("Failed to get free port pair for %s: %v", name, err))
-	}
-	env.closers = append(env.closers, httpCloser, grpcCloser)
-	return httpPort, grpcPort
-}
-
-func mustFreePort(t *testing.T, name string) int {
-	t.Helper()
-
-	port, closer, err := getFreePort()
-	if err != nil {
-		t.Fatalf("Failed to get free port for %s: %v", name, err)
-	}
-	closer.Close()
-	return port
-}
-
-func mustFreePortPair(t *testing.T, name string) (int, int) {
-	t.Helper()
-
-	httpPort, httpCloser, grpcPort, grpcCloser, err := findAvailablePortPair()
-	if err != nil {
-		t.Fatalf("Failed to get free port pair for %s: %v", name, err)
-	}
-	httpCloser.Close()
-	grpcCloser.Close()
-	return httpPort, grpcPort
-}
-
 func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	t.Helper()

@@ -198,12 +256,6 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	env.weedCancel = cancel

-	// Close all port listeners right before starting the weed process
-	for _, closer := range env.closers {
-		closer.Close()
-	}
-	env.closers = nil
-
 	cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
 		"-master.port", fmt.Sprintf("%d", env.masterPort),
 		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
@@ -332,18 +384,32 @@ func testIcebergRestAPI(t *testing.T, env *TestEnvironment) {
 	}
 }

-func (env *TestEnvironment) writeTrinoConfig(t *testing.T, warehouseBucket string) string {
+func (env *TestEnvironment) writeTrinoConfig(t *testing.T, warehouseBucket string, opts ...func(*trinoConfigOptions)) string {
 	t.Helper()

-	configDir := filepath.Join(env.dataDir, "trino")
+	o := trinoConfigOptions{}
+	for _, fn := range opts {
+		fn(&o)
+	}
+
+	dirName := "trino"
+	if o.nestedNamespace {
+		dirName = "trino-nested"
+	}
+	configDir := filepath.Join(env.dataDir, dirName)
 	if err := os.MkdirAll(configDir, 0755); err != nil {
 		t.Fatalf("Failed to create Trino config dir: %v", err)
 	}

+	nestedLine := ""
+	if o.nestedNamespace {
+		nestedLine = "\niceberg.rest-catalog.nested-namespace-enabled=true"
+	}
+
 	config := fmt.Sprintf(`connector.name=iceberg
 iceberg.catalog.type=rest
 iceberg.rest-catalog.uri=http://host.docker.internal:%d
-iceberg.rest-catalog.warehouse=s3://%s
+iceberg.rest-catalog.warehouse=s3://%s%s
 iceberg.file-format=PARQUET
 iceberg.unique-table-location=true

@@ -358,7 +424,7 @@ s3.region=us-west-2

 # REST catalog authentication
 iceberg.rest-catalog.security=SIGV4
-`, env.icebergPort, warehouseBucket, env.s3Port, env.accessKey, env.secretKey)
+`, env.icebergPort, warehouseBucket, nestedLine, env.s3Port, env.accessKey, env.secretKey)

 	if err := os.WriteFile(filepath.Join(configDir, "iceberg.properties"), []byte(config), 0644); err != nil {
 		t.Fatalf("Failed to write Trino config: %v", err)
@@ -367,6 +433,14 @@ iceberg.rest-catalog.security=SIGV4
 	return configDir
 }

+type trinoConfigOptions struct {
+	nestedNamespace bool
+}
+
+func withNestedNamespace() func(*trinoConfigOptions) {
+	return func(o *trinoConfigOptions) { o.nestedNamespace = true }
+}
+
 func (env *TestEnvironment) startTrinoContainer(t *testing.T, configDir string) {
 	t.Helper()

@@ -514,29 +588,6 @@ func hasDocker() bool {
 	return cmd.Run() == nil
 }

-func findAvailablePortPair() (int, io.Closer, int, io.Closer, error) {
-	httpPort, httpCloser, err := getFreePort()
-	if err != nil {
-		return 0, nil, 0, nil, err
-	}
-	grpcPort, grpcCloser, err := getFreePort()
-	if err != nil {
-		httpCloser.Close()
-		return 0, nil, 0, nil, err
-	}
-	return httpPort, httpCloser, grpcPort, grpcCloser, nil
-}
-
-func getFreePort() (int, io.Closer, error) {
-	listener, err := net.Listen("tcp", "0.0.0.0:0")
-	if err != nil {
-		return 0, nil, err
-	}
-
-	addr := listener.Addr().(*net.TCPAddr)
-	return addr.Port, listener, nil
-}
-
 func randomString(length int) string {
 	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
 	b := make([]byte, length)
@@ -25,7 +25,7 @@ import (
 	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/aws/aws-sdk-go-v2/service/sts"

-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
 )

@@ -16,7 +16,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"os"
 	"path"
@@ -40,6 +39,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"

+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/weed/command"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -92,7 +92,7 @@ func TestMain(m *testing.M) {
 }

 func startCluster(testDir string, extraArgs []string) (*testCluster, error) {
-	ports, err := findPorts(10)
+	ports, err := testutil.AllocatePorts(10)
 	if err != nil {
 		return nil, err
 	}
@@ -201,26 +201,6 @@ func (c *testCluster) filerConn(t *testing.T) (*grpc.ClientConn, filer_pb.Seawee
 	return conn, filer_pb.NewSeaweedFilerClient(conn)
 }

-func findPorts(n int) ([]int, error) {
-	ls := make([]*net.TCPListener, n)
-	ps := make([]int, n)
-	for i := 0; i < n; i++ {
-		l, err := net.Listen("tcp", "127.0.0.1:0")
-		if err != nil {
-			for j := 0; j < i; j++ {
-				ls[j].Close()
-			}
-			return nil, err
-		}
-		ls[i] = l.(*net.TCPListener)
-		ps[i] = ls[i].Addr().(*net.TCPAddr).Port
-	}
-	for _, l := range ls {
-		l.Close()
-	}
-	return ps, nil
-}
-
 func waitReady(endpoint string, timeout time.Duration) error {
 	client := &http.Client{Timeout: 1 * time.Second}
 	deadline := time.Now().Add(timeout)
@@ -19,7 +19,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"

-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )

 const (
@@ -9,7 +9,7 @@ import (
 	"testing"
 	"time"

-	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
 )

 // TestEnvironment mirrors the one in trino_catalog_test.go but simplified
@@ -3,7 +3,6 @@ package s3tables
 import (
 	"context"
 	"fmt"
-	"net"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -18,6 +17,7 @@ import (

 	"flag"

+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/weed/command"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
@@ -590,40 +590,12 @@ func testTargetOperations(t *testing.T, client *S3TablesClient) {

 // Helper functions

-// findAvailablePorts finds n available ports by binding to port 0 multiple times
-// It keeps the listeners open until all ports are found to ensure uniqueness
-func findAvailablePorts(n int) ([]int, error) {
-	listeners := make([]*net.TCPListener, n)
-	ports := make([]int, n)
-
-	// Open all listeners to ensure we get unique ports
-	for i := 0; i < n; i++ {
-		listener, err := net.Listen("tcp", "127.0.0.1:0")
-		if err != nil {
-			// Close valid listeners before returning error
-			for j := 0; j < i; j++ {
-				listeners[j].Close()
-			}
-			return nil, err
-		}
-		listeners[i] = listener.(*net.TCPListener)
-		ports[i] = listeners[i].Addr().(*net.TCPAddr).Port
-	}
-
-	// Close all listeners
-	for _, l := range listeners {
-		l.Close()
-	}
-
-	return ports, nil
-}
-
 // startMiniClusterInDir starts a weed mini instance using testDir as the data
 // directory. It does not require a *testing.T so it can be called from TestMain.
 // extraArgs are appended to the default mini command flags.
 func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, error) {
 	// We need 10 unique ports: Master(2), Volume(2), Filer(2), S3(2), Admin(2)
-	ports, err := findAvailablePorts(10)
+	ports, err := testutil.AllocatePorts(10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to find available ports: %v", err)
 	}
@@ -1,92 +0,0 @@
-package testutil
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"net/http"
-	"os/exec"
-	"testing"
-	"time"
-)
-
-func HasDocker() bool {
-	cmd := exec.Command("docker", "version")
-	return cmd.Run() == nil
-}
-
-// MustFreePortPair is a convenience wrapper for tests that only need a single pair.
-// Prefer MustAllocatePorts when allocating multiple pairs to guarantee uniqueness.
-func MustFreePortPair(t *testing.T, name string) (int, int) {
-	ports := MustAllocatePorts(t, 2)
-	return ports[0], ports[1]
-}
-
-// MustAllocatePorts allocates count unique free ports atomically.
-// All listeners are held open until every port is obtained, preventing
-// the OS from recycling a port between successive allocations.
-func MustAllocatePorts(t *testing.T, count int) []int {
-	t.Helper()
-	ports, err := AllocatePorts(count)
-	if err != nil {
-		t.Fatalf("Failed to allocate %d free ports: %v", count, err)
-	}
-	return ports
-}
-
-// AllocatePorts allocates count unique free ports atomically.
-func AllocatePorts(count int) ([]int, error) {
-	listeners := make([]net.Listener, 0, count)
-	ports := make([]int, 0, count)
-	for i := 0; i < count; i++ {
-		l, err := net.Listen("tcp", "0.0.0.0:0")
-		if err != nil {
-			for _, ll := range listeners {
-				_ = ll.Close()
-			}
-			return nil, err
-		}
-		listeners = append(listeners, l)
-		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
-	}
-	for _, l := range listeners {
-		_ = l.Close()
-	}
-	return ports, nil
-}
-
-func WaitForService(url string, timeout time.Duration) bool {
-	client := &http.Client{Timeout: 2 * time.Second}
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-
-	ticker := time.NewTicker(500 * time.Millisecond)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return false
-		case <-ticker.C:
-			resp, err := client.Get(url)
-			if err == nil {
-				resp.Body.Close()
-				return true
-			}
-		}
-	}
-}
-
-func WaitForPort(port int, timeout time.Duration) bool {
-	deadline := time.Now().Add(timeout)
-	address := fmt.Sprintf("127.0.0.1:%d", port)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", address, 500*time.Millisecond)
-		if err == nil {
-			_ = conn.Close()
-			return true
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-	return false
-}
@@ -1,17 +1,25 @@
 package testutil

 import (
+	"context"
 	"fmt"
-	"math/rand"
 	"net"
+	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
-	"testing"
 	"time"
 )

 const SeaweedMiniStartupTimeout = 45 * time.Second

+func HasDocker() bool {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "docker", "version")
+	return cmd.Run() == nil
+}
+
 func FindBindIP() string {
 	addrs, err := net.InterfaceAddrs()
 	if err != nil {
@@ -60,34 +68,38 @@ func WriteIAMConfig(dir, accessKey, secretKey string) (string, error) {
 	return iamConfigPath, nil
 }

-func MustFreeMiniPort(t *testing.T, name string) int {
-	t.Helper()
+func WaitForService(url string, timeout time.Duration) bool {
+	client := &http.Client{Timeout: 2 * time.Second}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()

-	const (
-		minPort = 10000
-		maxPort = 55000
-	)
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()

-	for i := 0; i < 1000; i++ {
-		port := minPort + r.Intn(maxPort-minPort)
-
-		listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
-		if err != nil {
-			continue
+	for {
+		select {
+		case <-ctx.Done():
+			return false
+		case <-ticker.C:
+			resp, err := client.Get(url)
+			if err == nil {
+				resp.Body.Close()
+				return true
+			}
 		}
-		_ = listener.Close()
-
-		grpcPort := port + 10000
-		grpcListener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
-		if err != nil {
-			continue
-		}
-		_ = grpcListener.Close()
-
-		return port
 	}
+}

-	t.Fatalf("failed to get free weed mini port for %s", name)
-	return 0
+func WaitForPort(port int, timeout time.Duration) bool {
+	deadline := time.Now().Add(timeout)
+	address := fmt.Sprintf("127.0.0.1:%d", port)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", address, 500*time.Millisecond)
+		if err == nil {
+			_ = conn.Close()
+			return true
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return false
 }
@@ -0,0 +1,120 @@
+// Package testutil provides shared test utilities for SeaweedFS integration tests.
+package testutil
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"testing"
+)
+
+// GrpcPortOffset is the offset weed mini uses to derive gRPC ports from HTTP ports.
+const GrpcPortOffset = 10000
+
+// AllocatePorts allocates count unique free ports atomically.
+// All listeners are held open until every port is obtained, preventing
+// the OS from recycling a port between successive allocations.
+func AllocatePorts(count int) ([]int, error) {
+	listeners := make([]net.Listener, 0, count)
+	ports := make([]int, 0, count)
+	for i := 0; i < count; i++ {
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			for _, ll := range listeners {
+				_ = ll.Close()
+			}
+			return nil, err
+		}
+		listeners = append(listeners, l)
+		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
+	}
+	for _, l := range listeners {
+		_ = l.Close()
+	}
+	return ports, nil
+}
+
+// MustAllocatePorts is a testing wrapper for AllocatePorts.
+func MustAllocatePorts(t *testing.T, count int) []int {
+	t.Helper()
+	ports, err := AllocatePorts(count)
+	if err != nil {
+		t.Fatalf("Failed to allocate %d free ports: %v", count, err)
+	}
+	return ports
+}
+
+// AllocateMiniPorts allocates n free ports where each port and its gRPC
+// counterpart (port + GrpcPortOffset) are available and don't collide
+// with any other allocated port or its gRPC counterpart. All listeners
+// are held open until the entire batch is allocated, preventing the OS
+// from recycling ports between allocations. Use this when ports will be
+// passed to weed mini without explicit gRPC port flags, so mini will
+// derive gRPC ports as HTTP + 10000.
+func AllocateMiniPorts(count int) ([]int, error) {
+	const (
+		minPort = 10000
+		maxPort = 55000
+	)
+	reserved := make(map[int]bool)
+	ports := make([]int, 0, count)
+	var listeners []net.Listener
+	defer func() {
+		for _, l := range listeners {
+			l.Close()
+		}
+	}()
+
+	for idx := 0; idx < count; idx++ {
+		found := false
+		for i := 0; i < 1000; i++ {
+			port := minPort + rand.Intn(maxPort-minPort)
+			grpcPort := port + GrpcPortOffset
+
+			if reserved[port] || reserved[grpcPort] {
+				continue
+			}
+
+			l1, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+			if err != nil {
+				continue
+			}
+
+			l2, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
+			if err != nil {
+				l1.Close()
+				continue
+			}
+
+			listeners = append(listeners, l1, l2)
+			reserved[port] = true
+			reserved[grpcPort] = true
+			ports = append(ports, port)
+			found = true
+			break
+		}
+		if !found {
+			return nil, fmt.Errorf("failed to allocate mini port %d of %d", idx+1, count)
+		}
+	}
+
+	return ports, nil
+}
+
+// MustFreeMiniPorts allocates n ports suitable for weed mini, ensuring
+// each port's gRPC offset (port + 10000) doesn't collide with any other
+// allocated port. names is used only for error messages.
+func MustFreeMiniPorts(t *testing.T, names []string) []int {
+	t.Helper()
+	ports, err := AllocateMiniPorts(len(names))
+	if err != nil {
+		t.Fatalf("failed to allocate mini ports for %v: %v", names, err)
+	}
+	return ports
+}
+
+// MustFreeMiniPort allocates a single weed mini port.
+func MustFreeMiniPort(t *testing.T, name string) int {
+	t.Helper()
+	return MustFreeMiniPorts(t, []string{name})[0]
+}
@@ -0,0 +1,402 @@
+package vacuum
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/shell"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+)
+
+type TestCluster struct {
+	masterCmd     *exec.Cmd
+	volumeServers []*exec.Cmd
+}
+
+func (c *TestCluster) Stop() {
+	for _, cmd := range c.volumeServers {
+		if cmd != nil && cmd.Process != nil {
+			cmd.Process.Kill()
+			cmd.Wait()
+		}
+	}
+	if c.masterCmd != nil && c.masterCmd.Process != nil {
+		c.masterCmd.Process.Kill()
+		c.masterCmd.Wait()
+	}
+}
+
+func startCluster(ctx context.Context, dataDir string) (*TestCluster, error) {
+	weedBinary := findWeedBinary()
+	if weedBinary == "" {
+		return nil, fmt.Errorf("weed binary not found - build with 'cd weed && go build' first")
+	}
+
+	cluster := &TestCluster{}
+
+	masterDir := filepath.Join(dataDir, "master")
+	os.MkdirAll(masterDir, 0755)
+
+	// Empty security.toml to disable JWT in tests
+	os.WriteFile(filepath.Join(dataDir, "security.toml"), []byte("# test\n"), 0644)
+
+	// Start master
+	masterCmd := exec.CommandContext(ctx, weedBinary, "master",
+		"-port", "9333",
+		"-mdir", masterDir,
+		"-volumeSizeLimitMB", "10",
+		"-ip", "127.0.0.1",
+	)
+	masterCmd.Dir = dataDir
+	masterLog, _ := os.Create(filepath.Join(masterDir, "master.log"))
+	masterCmd.Stdout = masterLog
+	masterCmd.Stderr = masterLog
+	if err := masterCmd.Start(); err != nil {
+		return nil, fmt.Errorf("start master: %v", err)
+	}
+	cluster.masterCmd = masterCmd
+	time.Sleep(2 * time.Second)
+
+	// Start 2 volume servers (enough for vacuum testing)
+	for i := 0; i < 2; i++ {
+		volumeDir := filepath.Join(dataDir, fmt.Sprintf("volume%d", i))
+		os.MkdirAll(volumeDir, 0755)
+
+		port := fmt.Sprintf("808%d", i)
+		volumeCmd := exec.CommandContext(ctx, weedBinary, "volume",
+			"-port", port,
+			"-dir", volumeDir,
+			"-max", "10",
+			"-master", "127.0.0.1:9333",
+			"-ip", "127.0.0.1",
+		)
+		volumeCmd.Dir = dataDir
+		volumeLog, _ := os.Create(filepath.Join(volumeDir, "volume.log"))
+		volumeCmd.Stdout = volumeLog
+		volumeCmd.Stderr = volumeLog
+		if err := volumeCmd.Start(); err != nil {
+			cluster.Stop()
+			return nil, fmt.Errorf("start volume server %d: %v", i, err)
+		}
+		cluster.volumeServers = append(cluster.volumeServers, volumeCmd)
+	}
+
+	time.Sleep(5 * time.Second)
+	return cluster, nil
+}
+
+func findWeedBinary() string {
+	candidates := []string{
+		"../../weed/weed",
+		"../weed/weed",
+		"./weed",
+	}
+	for _, c := range candidates {
+		if _, err := os.Stat(c); err == nil {
+			if abs, err := filepath.Abs(c); err == nil {
+				return abs
+			}
+			return c
+		}
+	}
+	if path, err := exec.LookPath("weed"); err == nil {
+		return path
+	}
+	return ""
+}
+
+func waitForServer(address string, timeout time.Duration) error {
+	start := time.Now()
+	for time.Since(start) < timeout {
+		if conn, err := net.DialTimeout("tcp", address, 1*time.Second); err == nil {
+			conn.Close()
+			return nil
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("timeout waiting for server %s", address)
+}
+
+func uploadData(masterAddr, collection string, data []byte) (string, needle.VolumeId, error) {
+	assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress {
+		return pb.ServerAddress(masterAddr)
+	}, grpc.WithInsecure(), &operation.VolumeAssignRequest{
+		Count:      1,
+		Collection: collection,
+	})
+	if err != nil {
+		return "", 0, fmt.Errorf("assign: %v", err)
+	}
+
+	uploader, err := operation.NewUploader()
+	if err != nil {
+		return "", 0, fmt.Errorf("new uploader: %v", err)
+	}
+
+	uploadResult, err, _ := uploader.Upload(context.Background(), bytes.NewReader(data), &operation.UploadOption{
+		UploadUrl: "http://" + assignResult.Url + "/" + assignResult.Fid,
+		Filename:  "testfile.txt",
+		MimeType:  "text/plain",
+	})
+	if err != nil {
+		return "", 0, fmt.Errorf("upload: %v", err)
+	}
+	if uploadResult.Error != "" {
+		return "", 0, fmt.Errorf("upload error: %s", uploadResult.Error)
+	}
+
+	fid, err := needle.ParseFileIdFromString(assignResult.Fid)
+	if err != nil {
+		return "", 0, err
+	}
+	return assignResult.Fid, fid.VolumeId, nil
+}
+
+func deleteFile(masterAddr string, fid string) error {
+	results := operation.DeleteFileIds(func(ctx context.Context) pb.ServerAddress {
+		return pb.ServerAddress(masterAddr)
+	}, false, grpc.WithInsecure(), []string{fid})
+	for _, r := range results {
+		if r.Error != "" {
+			return fmt.Errorf("delete %s: %s", fid, r.Error)
+		}
+	}
+	return nil
+}
+
+func getGarbageRatio(volumeServerAddr string, volumeId uint32) (float64, error) {
+	var ratio float64
+	err := operation.WithVolumeServerClient(false, pb.ServerAddress(volumeServerAddr), grpc.WithInsecure(),
+		func(client volume_server_pb.VolumeServerClient) error {
+			resp, err := client.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
+				VolumeId: volumeId,
+			})
+			if err != nil {
+				return err
+			}
+			ratio = resp.GarbageRatio
+			return nil
+		})
+	return ratio, err
+}
+
+// TestVacuumIntegration tests the full vacuum flow:
+// upload data → delete some → verify garbage → vacuum → verify cleanup
+func TestVacuumIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	testDir := t.TempDir()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	cluster, err := startCluster(ctx, testDir)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
+	require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second))
+	require.NoError(t, waitForServer("127.0.0.1:8081", 30*time.Second))
+
+	masterAddr := "127.0.0.1:9333"
+	collection := "vactest"
+
+	// Upload files large enough that deleting most creates significant garbage.
+	// With volumeSizeLimitMB=10, we need several MB of garbage to exceed the
+	// 10% threshold passed to vacuum.
+	const fileSize = 500 * 1024 // 500 KB per file
+	const totalFiles = 16
+	const filesToDelete = 12 // delete 75% → ~6 MB garbage out of ~8 MB
+
+	var fids []string
+	var payloads [][]byte
+	var volumeId needle.VolumeId
+	for i := 0; i < totalFiles; i++ {
+		data := bytes.Repeat([]byte{byte('A' + i%26)}, fileSize)
+		fid, vid, err := uploadData(masterAddr, collection, data)
+		require.NoError(t, err, "upload %d", i)
+		fids = append(fids, fid)
+		payloads = append(payloads, data)
+		volumeId = vid
+	}
+	t.Logf("Uploaded %d files (%d KB each) to volume %d", totalFiles, fileSize/1024, volumeId)
+
+	// Wait for heartbeat to report sizes
+	time.Sleep(6 * time.Second)
+
+	// Delete most files to create garbage well above the threshold
+	for i := 0; i < filesToDelete; i++ {
+		err := deleteFile(masterAddr, fids[i])
+		require.NoError(t, err, "delete %s", fids[i])
+	}
+	t.Logf("Deleted %d of %d files to create garbage", filesToDelete, totalFiles)
+
+	// Wait for heartbeat to report deletions
+	time.Sleep(6 * time.Second)
+
+	// Verify garbage exists
+	t.Run("verify_garbage_before_vacuum", func(t *testing.T) {
+		for _, addr := range []string{"127.0.0.1:8080", "127.0.0.1:8081"} {
+			ratio, err := getGarbageRatio(addr, uint32(volumeId))
+			if err != nil {
+				continue
+			}
+			t.Logf("Garbage ratio on %s: %.2f%%", addr, ratio*100)
+			if ratio > 0.1 {
+				return // sufficient garbage found
+			}
+		}
+		t.Fatal("No server reported garbage > 10% — test data setup failed")
+	})
+
+	// Execute vacuum via shell command
+	t.Run("run_vacuum", func(t *testing.T) {
+		options := &shell.ShellOptions{
+			Masters:        stringPtr(masterAddr),
+			GrpcDialOption: grpc.WithInsecure(),
+			FilerGroup:     stringPtr("default"),
+		}
+		commandEnv := shell.NewCommandEnv(options)
+
+		shellCtx, shellCancel := context.WithTimeout(context.Background(), 60*time.Second)
+		defer shellCancel()
+		go commandEnv.MasterClient.KeepConnectedToMaster(shellCtx)
+		commandEnv.MasterClient.WaitUntilConnected(shellCtx)
+		time.Sleep(2 * time.Second)
+
+		// Acquire lock (required by shell commands)
+		locked, unlock := tryLock(t, commandEnv, 30*time.Second)
+		require.True(t, locked, "could not acquire shell lock")
+		defer unlock()
+
+		// Find and execute vacuum command
+		var output bytes.Buffer
+		var found bool
+		var err error
+		for _, cmd := range shell.Commands {
+			if cmd.Name() == "volume.vacuum" {
+				err = cmd.Do(
+					[]string{"-garbageThreshold", "0.1", "-collection", collection},
+					commandEnv, &output,
+				)
+				found = true
+				break
+			}
+		}
+		require.True(t, found, "volume.vacuum command not found")
+		t.Logf("Vacuum output: %s", output.String())
+		require.NoError(t, err, "vacuum command failed")
+		t.Log("Vacuum completed successfully")
+	})
+
+	// Wait for vacuum effects to settle
+	time.Sleep(6 * time.Second)
+
+	// Verify garbage was cleaned
+	t.Run("verify_cleanup_after_vacuum", func(t *testing.T) {
+		var volumeFound, cleanupVerified bool
+		for _, addr := range []string{"127.0.0.1:8080", "127.0.0.1:8081"} {
+			ratio, err := getGarbageRatio(addr, uint32(volumeId))
+			if err != nil {
+				continue
+			}
+			volumeFound = true
+			t.Logf("Garbage ratio after vacuum on %s: %.2f%%", addr, ratio*100)
+			if ratio < 0.05 {
+				cleanupVerified = true
+			}
+		}
+		if !volumeFound {
+			t.Fatal("No server reported volume after vacuum")
+		}
+		if !cleanupVerified {
+			t.Fatal("Garbage was not cleaned up after vacuum")
+		}
+	})
+
+	// Verify remaining files are still readable with correct contents
+	t.Run("verify_remaining_data", func(t *testing.T) {
+		for i := filesToDelete; i < totalFiles; i++ {
+			fid := fids[i]
+			expected := payloads[i]
+
+			// Read file via HTTP from volume server
+			client := &http.Client{Timeout: 5 * time.Second}
+			url := fmt.Sprintf("http://127.0.0.1:8080/%s", fid)
+			resp, err := client.Get(url)
+			if err != nil || resp.StatusCode == http.StatusNotFound {
+				if resp != nil {
+					resp.Body.Close()
+				}
+				url = fmt.Sprintf("http://127.0.0.1:8081/%s", fid)
+				resp, err = client.Get(url)
+			}
+			require.NoError(t, err, "read fid %s", fid)
+			body, err := io.ReadAll(resp.Body)
+			resp.Body.Close()
+			require.NoError(t, err, "read body of fid %s", fid)
+			require.Equal(t, http.StatusOK, resp.StatusCode, "fid %s returned %d", fid, resp.StatusCode)
+			require.Equal(t, len(expected), len(body), "fid %s size mismatch", fid)
+			require.True(t, bytes.Equal(expected, body), "fid %s content mismatch", fid)
+			t.Logf("File %s verified (%d bytes)", fid, len(body))
+		}
+	})
+}
+
+func stringPtr(s string) *string {
+	return &s
+}
+
+func tryLock(t *testing.T, commandEnv *shell.CommandEnv, timeout time.Duration) (locked bool, unlock func()) {
+	t.Helper()
+	type result struct {
+		err error
+	}
+	done := make(chan result, 1)
+	go func() {
+		for _, cmd := range shell.Commands {
+			if cmd.Name() == "lock" {
+				var out bytes.Buffer
+				done <- result{err: cmd.Do([]string{}, commandEnv, &out)}
+				return
+			}
+		}
+		done <- result{err: fmt.Errorf("lock command not found")}
+	}()
+
+	select {
+	case res := <-done:
+		if res.err != nil {
+			t.Logf("lock failed: %v", res.err)
+			return false, nil
+		}
+		return true, func() {
+			for _, cmd := range shell.Commands {
+				if cmd.Name() == "unlock" {
+					var out bytes.Buffer
+					cmd.Do([]string{}, commandEnv, &out)
+					return
+				}
+			}
+		}
+	case <-time.After(timeout):
+		t.Log("lock timed out")
+		return false, nil
+	}
+}
@@ -18,6 +18,7 @@ import (
 	"testing"
 	"time"

+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 )

@@ -84,12 +85,14 @@ func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
 		t.Fatalf("write security config: %v", err)
 	}

-	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	miniPorts, err := testutil.AllocateMiniPorts(1)
 	if err != nil {
 		t.Fatalf("allocate master port pair: %v", err)
 	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset

-	ports, err := allocatePorts(3)
+	ports, err := testutil.AllocatePorts(3)
 	if err != nil {
 		t.Fatalf("allocate ports: %v", err)
 	}
@@ -269,45 +272,6 @@ func stopProcess(cmd *exec.Cmd) {
 	}
 }

-func allocatePorts(count int) ([]int, error) {
-	listeners := make([]net.Listener, 0, count)
-	ports := make([]int, 0, count)
-	for i := 0; i < count; i++ {
-		l, err := net.Listen("tcp", "127.0.0.1:0")
-		if err != nil {
-			for _, ll := range listeners {
-				_ = ll.Close()
-			}
-			return nil, err
-		}
-		listeners = append(listeners, l)
-		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
-	}
-	for _, l := range listeners {
-		_ = l.Close()
-	}
-	return ports, nil
-}
-
-func allocateMasterPortPair() (int, int, error) {
-	for masterPort := 10000; masterPort <= 55535; masterPort++ {
-		masterGrpcPort := masterPort + 10000
-		l1, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterPort)))
-		if err != nil {
-			continue
-		}
-		l2, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterGrpcPort)))
-		if err != nil {
-			_ = l1.Close()
-			continue
-		}
-		_ = l2.Close()
-		_ = l1.Close()
-		return masterPort, masterGrpcPort, nil
-	}
-	return 0, 0, errors.New("unable to find available master port pair")
-}
-
 func newWorkDir() (dir string, keepLogs bool, err error) {
 	keepLogs = os.Getenv("VOLUME_SERVER_IT_KEEP_LOGS") == "1"
 	dir, err = os.MkdirTemp("", "seaweedfs_volume_server_it_")
@@ -10,6 +10,7 @@ import (
 	"sync"
 	"testing"

+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 )

@@ -92,17 +93,19 @@ func StartMixedVolumeCluster(t testing.TB, profile matrix.Profile, goCount, rust
 		t.Fatalf("write security config: %v", err)
 	}

-	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	miniPorts, err := testutil.AllocateMiniPorts(1)
 	if err != nil {
 		t.Fatalf("allocate master port pair: %v", err)
 	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset

 	// 2 ports per server (admin, grpc); add 1 more when public port is split out.
 	portsPerServer := 2
 	if profile.SplitPublicPort {
 		portsPerServer = 3
 	}
-	ports, err := allocatePorts(total * portsPerServer)
+	ports, err := testutil.AllocatePorts(total * portsPerServer)
 	if err != nil {
 		t.Fatalf("allocate volume ports: %v", err)
 	}
@@ -10,6 +10,7 @@ import (
 	"sync"
 	"testing"

+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 )

@@ -74,10 +75,12 @@ func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount i
 		t.Fatalf("write security config: %v", err)
 	}

-	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	miniPorts, err := testutil.AllocateMiniPorts(1)
 	if err != nil {
 		t.Fatalf("allocate master port pair: %v", err)
 	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset

 	// Allocate ports for all volume servers (3 ports per server: admin, grpc, public)
 	// If SplitPublicPort is true, we need an additional port per server
@@ -86,7 +89,7 @@ func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount i
 		portsPerServer = 4
 	}
 	totalPorts := serverCount * portsPerServer
-	ports, err := allocatePorts(totalPorts)
+	ports, err := testutil.AllocatePorts(totalPorts)
 	if err != nil {
 		t.Fatalf("allocate volume ports: %v", err)
 	}
@@ -10,6 +10,7 @@ import (
 	"sync"
 	"testing"

+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 )

@@ -85,10 +86,12 @@ func StartRustMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCou
 		t.Fatalf("write security config: %v", err)
 	}

-	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	miniPorts, err := testutil.AllocateMiniPorts(1)
 	if err != nil {
 		t.Fatalf("allocate master port pair: %v", err)
 	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset

 	// Allocate ports for all volume servers (3 ports per server: admin, grpc, public)
 	// If SplitPublicPort is true, we need an additional port per server
@@ -97,7 +100,7 @@ func StartRustMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCou
 		portsPerServer = 4
 	}
 	totalPorts := serverCount * portsPerServer
-	ports, err := allocatePorts(totalPorts)
+	ports, err := testutil.AllocatePorts(totalPorts)
 	if err != nil {
 		t.Fatalf("allocate volume ports: %v", err)
 	}
@@ -12,6 +12,7 @@ import (
 	"sync"
 	"testing"

+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 )

@@ -79,12 +80,14 @@ func StartRustVolumeCluster(t testing.TB, profile matrix.Profile) *RustCluster {
 		t.Fatalf("write security config: %v", err)
 	}

-	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	miniPorts, err := testutil.AllocateMiniPorts(1)
 	if err != nil {
 		t.Fatalf("allocate master port pair: %v", err)
 	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset

-	ports, err := allocatePorts(3)
+	ports, err := testutil.AllocatePorts(3)
 	if err != nil {
 		t.Fatalf("allocate ports: %v", err)
 	}
@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"testing"

+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 )

@@ -25,7 +26,7 @@ func StartSingleVolumeClusterWithFiler(t testing.TB, profile matrix.Profile) *Cl

 	baseCluster := StartSingleVolumeCluster(t, profile)

-	ports, err := allocatePorts(2)
+	ports, err := testutil.AllocatePorts(2)
 	if err != nil {
 		t.Fatalf("allocate filer ports: %v", err)
 	}
@@ -80,6 +80,42 @@ func EnableMaintenanceMode(t testing.TB, ctx context.Context, client volume_serv
 	}
 }

+// CorruptDatFile truncates a volume's .dat file to just the superblock (8 bytes)
+// so that needle reads fail during a full scrub due to data file size mismatch.
+func CorruptDatFile(t testing.TB, baseDir string, volumeID uint32) {
+	t.Helper()
+	datPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.dat", volumeID))
+	// Truncate to superblock size only, removing all needle data.
+	if err := os.Truncate(datPath, 8); err != nil {
+		t.Fatalf("truncate dat file for corruption: %v", err)
+	}
+}
+
+// CorruptEcxFile appends garbage bytes to a volume's .ecx file on disk so
+// that CheckIndexFile detects a size mismatch during EC index scrub.
+func CorruptEcxFile(t testing.TB, baseDir string, volumeID uint32) {
+	t.Helper()
+	ecxPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.ecx", volumeID))
+	f, err := os.OpenFile(ecxPath, os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		t.Fatalf("open ecx file for corruption: %v", err)
+	}
+	defer f.Close()
+	if _, err := f.Write([]byte{0xDE, 0xAD}); err != nil {
+		t.Fatalf("corrupt ecx file: %v", err)
+	}
+}
+
+// CorruptEcShardFile truncates an EC shard file to 1 byte so that local shard
+// reads fail during an EC scrub.
+func CorruptEcShardFile(t testing.TB, baseDir string, volumeID uint32, shardID int) {
+	t.Helper()
+	shardPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.ec%02d", volumeID, shardID))
+	if err := os.Truncate(shardPath, 1); err != nil {
+		t.Fatalf("truncate EC shard file %s: %v", shardPath, err)
+	}
+}
+
 func ReadBytes(t testing.TB, client *http.Client, volumeURL, fid string) *http.Response {
 	t.Helper()

@@ -752,6 +752,87 @@ func TestEcShardsCopyFromPeerSuccess(t *testing.T) {
 	}
 }

+// TestEcIndexConsistencyAfterEncode verifies that every needle indexed in .ecx
+// can be read back correctly from EC shards after VolumeEcShardsGenerate.
+// This catches the race condition fixed in this PR where .ecx could reference
+// data not present in EC shards.
+func TestEcIndexConsistencyAfterEncode(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(130)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+
+	// Upload multiple needles of varying sizes
+	type testNeedle struct {
+		fid     string
+		payload []byte
+	}
+	needles := []testNeedle{
+		{framework.NewFileID(volumeID, 1001, 0xAABB0001), []byte("small-needle-1")},
+		{framework.NewFileID(volumeID, 1002, 0xAABB0002), make([]byte, 1024)},     // 1KB
+		{framework.NewFileID(volumeID, 1003, 0xAABB0003), make([]byte, 64*1024)},  // 64KB
+		{framework.NewFileID(volumeID, 1004, 0xAABB0004), make([]byte, 256*1024)}, // 256KB
+		{framework.NewFileID(volumeID, 1005, 0xAABB0005), []byte("small-needle-2")},
+	}
+
+	// Fill larger payloads with recognizable data
+	for i := range needles {
+		for j := range needles[i].payload {
+			needles[i].payload[j] = byte(i*37 + j%251)
+		}
+	}
+
+	for _, n := range needles {
+		resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload)
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s expected 201, got %d", n.fid, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// EC encode
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	// Mount all data shards so reads go through the EC path
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount failed: %v", err)
+	}
+
+	// Read every needle back from EC shards and verify payload
+	for _, n := range needles {
+		readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid)
+		readBody := framework.ReadAllAndClose(t, readResp)
+		if readResp.StatusCode != http.StatusOK {
+			t.Fatalf("EC read %s expected 200, got %d", n.fid, readResp.StatusCode)
+		}
+		if string(readBody) != string(n.payload) {
+			t.Fatalf("EC read %s payload mismatch: got %d bytes, want %d bytes", n.fid, len(readBody), len(n.payload))
+		}
+	}
+}
+
 func TestEcShardsCopyFailsWhenSourceUnavailable(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test in short mode")
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -17,6 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"

+	"github.com/seaweedfs/seaweedfs/test/testutil"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
@@ -24,29 +24,6 @@ import (
 )

 // findAvailablePort finds a free TCP port on localhost.
-func findAvailablePort() (int, error) {
-	l, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		return 0, err
-	}
-	port := l.Addr().(*net.TCPAddr).Port
-	l.Close()
-	return port, nil
-}
-
-// waitForPort waits until a TCP port is listening, up to timeout.
-func waitForPort(port int, timeout time.Duration) error {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 500*time.Millisecond)
-		if err == nil {
-			conn.Close()
-			return nil
-		}
-		time.Sleep(200 * time.Millisecond)
-	}
-	return fmt.Errorf("port %d not listening after %v", port, timeout)
-}

 // startWeedMini starts a weed mini subprocess and returns the S3 endpoint and cleanup func.
 func startWeedMini(t *testing.T) (s3Endpoint string, cleanup func()) {
@@ -60,10 +37,11 @@ func startWeedMini(t *testing.T) (s3Endpoint string, cleanup func()) {
 		}
 	}

-	miniMasterPort, _ := findAvailablePort()
-	miniVolumePort, _ := findAvailablePort()
-	miniFilerPort, _ := findAvailablePort()
-	miniS3Port, _ := findAvailablePort()
+	miniPorts := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3"})
+	miniMasterPort := miniPorts[0]
+	miniVolumePort := miniPorts[1]
+	miniFilerPort := miniPorts[2]
+	miniS3Port := miniPorts[3]
 	miniDir := t.TempDir()
 	os.WriteFile(filepath.Join(miniDir, "security.toml"), []byte("# empty\n"), 0644)

@@ -89,11 +67,11 @@ func startWeedMini(t *testing.T) (s3Endpoint string, cleanup func()) {
 		t.Fatalf("start weed mini: %v", err)
 	}

-	if err := waitForPort(miniS3Port, 30*time.Second); err != nil {
+	if !testutil.WaitForPort(miniS3Port, 30*time.Second) {
 		cancel()
 		miniCmd.Wait()
 		logFile.Close()
-		t.Fatalf("weed mini S3 not ready: %v", err)
+		t.Fatalf("weed mini S3 not ready on port %d", miniS3Port)
 	}
 	t.Logf("weed mini S3 ready on port %d", miniS3Port)

@@ -0,0 +1,416 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// --- Normal volume scrub tests ---
+
+func TestScrubVolumeFullHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(200)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("data-one")))
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 2, 2), []byte("data-two")))
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 3, 3), []byte("data-three")))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume FULL on healthy volume failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 1 {
+		t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
+	}
+	if resp.GetTotalFiles() != 3 {
+		t.Fatalf("expected total_files=3, got %d", resp.GetTotalFiles())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
+	}
+}
+
+func TestScrubVolumeFullCorruptData(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(201)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("important data")))
+
+	framework.CorruptDatFile(t, clusterHarness.BaseDir(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume FULL on corrupt volume failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after data corruption, got none")
+	}
+	if len(resp.GetDetails()) == 0 {
+		t.Fatalf("expected error details for corrupt volume")
+	}
+}
+
+func TestScrubVolumeMixedHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const healthyVol = uint32(202)
+	const corruptVol = uint32(203)
+	framework.AllocateVolume(t, grpcClient, healthyVol, "")
+	framework.AllocateVolume(t, grpcClient, corruptVol, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(healthyVol, 1, 1), []byte("healthy")))
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(corruptVol, 1, 1), []byte("will corrupt")))
+
+	framework.CorruptIndexFile(t, clusterHarness.BaseDir(), corruptVol)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{healthyVol, corruptVol},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume INDEX on mixed volumes failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 2 {
+		t.Fatalf("expected total_volumes=2, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 1 {
+		t.Fatalf("expected exactly 1 broken volume, got %v", resp.GetBrokenVolumeIds())
+	}
+	if resp.GetBrokenVolumeIds()[0] != corruptVol {
+		t.Fatalf("expected broken volume %d, got %d", corruptVol, resp.GetBrokenVolumeIds()[0])
+	}
+}
+
+func TestScrubVolumeMissingVolumeReturnsError(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{99999},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err == nil {
+		t.Fatalf("ScrubVolume should fail for missing volume")
+	}
+}
+
+// --- EC volume scrub tests ---
+
+// ecSetup creates a volume, uploads data, generates EC shards, and mounts all of them.
+func ecSetup(t *testing.T, grpcClient volume_server_pb.VolumeServerClient, httpClient *http.Client, volumeURL string, volumeID uint32) {
+	t.Helper()
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 1, 0xABCD0001)
+	uploadResp := framework.UploadBytes(t, httpClient, volumeURL, fid, []byte("ec-scrub-test-data-payload"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	allShards := make([]uint32, erasure_coding.TotalShardsCount)
+	for i := range allShards {
+		allShards[i] = uint32(i)
+	}
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   allShards,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount all shards failed: %v", err)
+	}
+}
+
+func TestScrubEcVolumeIndexHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(210)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume INDEX on healthy volume failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 1 {
+		t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
+	}
+}
+
+func TestScrubEcVolumeLocalHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(211)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_LOCAL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume LOCAL on healthy volume failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 1 {
+		t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
+	}
+	if resp.GetTotalFiles() != 1 {
+		t.Fatalf("expected total_files=1, got %d", resp.GetTotalFiles())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
+	}
+	if len(resp.GetBrokenShardInfos()) != 0 {
+		t.Fatalf("expected no broken shards, got %v", resp.GetBrokenShardInfos())
+	}
+}
+
+func TestScrubEcVolumeLocalCorruptShard(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(212)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	// Corrupt shard 0 by truncating it.
+	framework.CorruptEcShardFile(t, clusterHarness.BaseDir(), volumeID, 0)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_LOCAL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume LOCAL on corrupt shard failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after shard corruption")
+	}
+	if len(resp.GetBrokenShardInfos()) == 0 {
+		t.Fatalf("expected broken shard info after shard corruption")
+	}
+	// Verify all reported broken shards belong to the corrupted volume.
+	for _, si := range resp.GetBrokenShardInfos() {
+		if si.GetVolumeId() != volumeID {
+			t.Fatalf("broken shard info for unexpected volume %d, want %d", si.GetVolumeId(), volumeID)
+		}
+	}
+}
+
+func TestScrubEcVolumeAutoSelectWithEcPresent(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeIDA = uint32(213)
+	const volumeIDB = uint32(214)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeIDA)
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeIDB)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// Auto-select (empty VolumeIds) should find both EC volumes.
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		Mode: volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume auto-select failed: %v", err)
+	}
+	if resp.GetTotalVolumes() < 2 {
+		t.Fatalf("expected at least 2 EC volumes via auto-select, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v", resp.GetBrokenVolumeIds())
+	}
+}
+
+func TestScrubEcVolumeUnsupportedMode(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(215)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode(99),
+	})
+	if err == nil {
+		t.Fatalf("ScrubEcVolume should fail for unsupported mode")
+	}
+}
+
+func TestScrubEcVolumeIndexCorruptEcx(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(216)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 1, 0xABCD0001)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-ecx-corrupt-test"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// Generate EC shards (this creates the .ecx file on disk).
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	// Corrupt the .ecx file BEFORE mounting, so the corrupted size is loaded.
+	framework.CorruptEcxFile(t, clusterHarness.BaseDir(), volumeID)
+
+	// Now mount shards - the ecx file size will reflect the corruption.
+	allShards := make([]uint32, erasure_coding.TotalShardsCount)
+	for i := range allShards {
+		allShards[i] = uint32(i)
+	}
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   allShards,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount failed: %v", err)
+	}
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume INDEX on corrupt ecx failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after ECX corruption")
+	}
+}
@@ -459,9 +459,10 @@ func sortDurations(d []time.Duration) {
 // This reveals tail latency differences that short tests miss (GC pauses, lock contention, etc).
 //
 // Run:
-//   go test -v -count=1 -timeout 600s -run TestSustainedP99 ./test/volume_server/loadtest/...
-//   VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 600s -run TestSustainedP99 ./test/volume_server/loadtest/...
-//   LOADTEST_DURATION=120s VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 600s -run TestSustainedP99 ./test/volume_server/loadtest/...
+//
+//	go test -v -count=1 -timeout 600s -run TestSustainedP99 ./test/volume_server/loadtest/...
+//	VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 600s -run TestSustainedP99 ./test/volume_server/loadtest/...
+//	LOADTEST_DURATION=120s VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 600s -run TestSustainedP99 ./test/volume_server/loadtest/...
 func TestSustainedP99(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping sustained load test in short mode")
@@ -289,8 +289,8 @@ func (cp *ConfigPersistence) LoadVacuumTaskConfig() (*VacuumTaskConfig, error) {

 	// Return default config if no valid config found
 	return &VacuumTaskConfig{
-		GarbageThreshold:   0.3,
-		MinVolumeAgeHours:  24,
+		GarbageThreshold:  0.3,
+		MinVolumeAgeHours: 24,
 	}, nil
 }

@@ -305,8 +305,8 @@ func (cp *ConfigPersistence) LoadVacuumTaskPolicy() (*worker_pb.TaskPolicy, erro
 			CheckIntervalSeconds:  6 * 3600,  // 6 hours in seconds
 			TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
 				VacuumConfig: &worker_pb.VacuumTaskConfig{
-					GarbageThreshold:   0.3,
-					MinVolumeAgeHours:  24,
+					GarbageThreshold:  0.3,
+					MinVolumeAgeHours: 24,
 				},
 			},
 		}, nil
@@ -325,8 +325,8 @@ func (cp *ConfigPersistence) LoadVacuumTaskPolicy() (*worker_pb.TaskPolicy, erro
 			CheckIntervalSeconds:  6 * 3600,  // 6 hours in seconds
 			TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
 				VacuumConfig: &worker_pb.VacuumTaskConfig{
-					GarbageThreshold:   0.3,
-					MinVolumeAgeHours:  24,
+					GarbageThreshold:  0.3,
+					MinVolumeAgeHours: 24,
 				},
 			},
 		}, nil
@@ -705,8 +705,8 @@ func buildPolicyFromTaskConfigs() *worker_pb.MaintenancePolicy {
 			CheckIntervalSeconds:  int32(vacuumConfig.ScanIntervalSeconds),
 			TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
 				VacuumConfig: &worker_pb.VacuumTaskConfig{
-					GarbageThreshold:   float64(vacuumConfig.GarbageThreshold),
-					MinVolumeAgeHours:  int32(vacuumConfig.MinVolumeAgeSeconds / 3600), // Convert seconds to hours
+					GarbageThreshold:  float64(vacuumConfig.GarbageThreshold),
+					MinVolumeAgeHours: int32(vacuumConfig.MinVolumeAgeSeconds / 3600), // Convert seconds to hours
 				},
 			},
 		}
@@ -452,6 +452,13 @@ func (s *AdminServer) UpdatePluginJobTypeConfigAPI(w http.ResponseWriter, r *htt
 	}
 	config.UpdatedBy = username

+	// Reapply descriptor defaults so a save from an older form (or a UI
+	// that omits new fields) cannot silently clear a baseline like
+	// execution_timeout_seconds back to zero.
+	if descriptor, err := s.LoadPluginJobTypeDescriptor(jobType); err == nil && descriptor != nil {
+		applyDescriptorDefaultsToPersistedConfig(config, descriptor)
+	}
+
 	if err := s.SavePluginJobTypeConfig(config); err != nil {
 		writeJSONError(w, http.StatusInternalServerError, err.Error())
 		return
@@ -916,6 +923,9 @@ func applyDescriptorDefaultsToPersistedConfig(
 		if runtime.JobTypeMaxRuntimeSeconds <= 0 {
 			runtime.JobTypeMaxRuntimeSeconds = defaults.JobTypeMaxRuntimeSeconds
 		}
+		if runtime.ExecutionTimeoutSeconds <= 0 {
+			runtime.ExecutionTimeoutSeconds = defaults.ExecutionTimeoutSeconds
+		}
 		if runtime.RetryBackoffSeconds <= 0 {
 			runtime.RetryBackoffSeconds = defaults.RetryBackoffSeconds
 		}
--- a/Show More
+++ b/Show More