feat: Add ID option to plugin_worker command

- Add -id CLI flag to allow custom worker IDs
- Auto-generates ID if not specified (worker-{hostname}-{timestamp})
- ID defaults to empty string and is optional
- Log the custom ID when specified
- Update usage examples to show -id option
- Worker can now be identified by custom name in admin UI

.github/workflows/container_release_unified.yml

@@ -261,7 +261,7 @@ jobs:
   helm-release:
     runs-on: ubuntu-latest
     needs: [copy-to-dockerhub]
-    if: github.event_name == 'push'
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     permissions:
       contents: write
       pages: write

.github/workflows/go.yml

@@ -15,24 +15,20 @@ permissions:
 
 jobs:
 
-  build:
-    name: Build
+  vet:
+    name: Go Vet
     runs-on: ubuntu-latest
     steps:
-
     - name: Set up Go 1.x
       uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
       with:
         go-version: ^1.13
       id: go
-
     - name: Check out code into the Go module directory
       uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
-
     - name: Get dependencies
       run: |
         cd weed; go get -v -t -d ./...
-
     - name: Go Vet (excluding protobuf lock copying)
       run: |
         cd weed
@@ -42,8 +38,30 @@ jobs:
         # Fail only if there are actual vet errors (not counting the filtered lock warnings)
         if grep -q "vet:" vet-output.txt; then exit 1; fi
 
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
+      with:
+        go-version: ^1.13
+      id: go
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
     - name: Build
       run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
 
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
+      with:
+        go-version: ^1.13
+      id: go
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
     - name: Test
       run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...

.github/workflows/helm_manual_release.yml

@@ -0,0 +1,25 @@
+name: "helm: manual release"
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pages: write
+
+jobs:
+  helm-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Publish Helm charts
+        uses: stefanprodan/helm-gh-pages@v1.7.0
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          charts_dir: k8s/charts
+          target_dir: helm
+          branch: gh-pages
+          helm_version: "3.18.4"

.github/workflows/java_integration_tests.yml

@@ -111,7 +111,7 @@ jobs:
           
           # Wait for S3 API
           for i in {1..30}; do
-            if curl -s http://localhost:8333/ > /dev/null 2>&1; then
+            if curl -s http://localhost:8333/healthz > /dev/null 2>&1; then
               echo "✓ S3 API is ready"
               break
             fi

.github/workflows/s3-go-tests.yml

@@ -461,6 +461,8 @@ jobs:
           export S3_ENDPOINT="http://localhost:8006"
           export S3_ACCESS_KEY="0555b35654ad1656d804"
           export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
+          export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY"
+          export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY"
           
           # Run the specific test that is equivalent to AWS S3 tagging behavior
           make test-with-server || {

.github/workflows/s3-keycloak-tests.yml

@@ -97,7 +97,7 @@ jobs:
           # Verify service accessibility
           echo "=== Verifying Service Accessibility ==="
           curl -f http://localhost:8080/realms/master
-          curl -s http://localhost:8333
+          curl -s http://localhost:8333/healthz
           echo "✅ SeaweedFS S3 API is responding (IAM-protected endpoint)"
           
           # Run Keycloak-specific tests

.github/workflows/s3-tables-tests.yml

@@ -260,6 +260,209 @@ jobs:
           path: test/s3tables/catalog_spark/test-output.log
           retention-days: 3
 
+  risingwave-catalog-tests:
+    name: RisingWave Catalog Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v3
+
+      - name: Pre-pull RisingWave image
+        run: |
+          docker pull risingwavelabs/risingwave:v2.5.0
+          docker pull postgres:16-alpine
+
+      - name: Run go mod tidy
+        run: go mod tidy
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false ./weed
+
+      - name: Run RisingWave Catalog Integration Tests
+        timeout-minutes: 25
+        working-directory: test/s3tables/catalog_risingwave
+        run: |
+          set -x
+          set -o pipefail
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          df -h
+          echo "=== Starting RisingWave Catalog Tests ==="
+          
+          # Run RisingWave catalog integration tests
+          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
+            echo "RisingWave catalog integration tests failed"
+            exit 1
+          }
+
+      - name: Show test output on failure
+        if: failure()
+        working-directory: test/s3tables/catalog_risingwave
+        run: |
+          echo "=== Test Output ==="
+          if [ -f test-output.log ]; then
+            tail -200 test-output.log
+          fi
+          
+          echo "=== Process information ==="
+          ps aux | grep -E "(weed|test|docker)" || true
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: risingwave-catalog-test-logs
+          path: test/s3tables/catalog_risingwave/test-output.log
+          retention-days: 3
+
+  sts-integration-tests:
+    name: STS Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v3
+
+      - name: Pre-pull Python image
+        run: docker pull python:3
+
+      - name: Run go mod tidy
+        run: go mod tidy
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false ./weed
+
+      - name: Run STS Integration Tests
+        timeout-minutes: 25
+        working-directory: test/s3tables/sts_integration
+        run: |
+          set -x
+          set -o pipefail
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          df -h
+          echo "=== Starting STS Integration Tests ==="
+          
+          # Run STS integration tests
+          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
+            echo "STS integration tests failed"
+            exit 1
+          }
+
+      - name: Show test output on failure
+        if: failure()
+        working-directory: test/s3tables/sts_integration
+        run: |
+          echo "=== Test Output ==="
+          if [ -f test-output.log ]; then
+            tail -200 test-output.log
+          fi
+          
+          echo "=== Process information ==="
+          ps aux | grep -E "(weed|test|docker)" || true
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: sts-integration-test-logs
+          path: test/s3tables/sts_integration/test-output.log
+          retention-days: 3
+
+  lakekeeper-integration-tests:
+    name: Lakekeeper Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v3
+
+      - name: Pre-pull Python image
+        run: docker pull python:3
+        
+      - name: Pre-pull LocalStack image (if needed)
+        run: docker pull localstack/localstack:latest || true
+
+      - name: Run go mod tidy
+        run: go mod tidy
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false ./weed
+
+      - name: Run Lakekeeper Integration Tests
+        timeout-minutes: 25
+        working-directory: test/s3tables/lakekeeper
+        run: |
+          set -x
+          set -o pipefail
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          df -h
+          echo "=== Starting Lakekeeper Integration Tests ==="
+          
+          # Run Lakekeeper integration tests
+          go test -v -timeout 20m . 2>&1 | tee test-output.log || {
+            echo "Lakekeeper integration tests failed"
+            exit 1
+          }
+
+      - name: Show test output on failure
+        if: failure()
+        working-directory: test/s3tables/lakekeeper
+        run: |
+          echo "=== Test Output ==="
+          if [ -f test-output.log ]; then
+            tail -200 test-output.log
+          fi
+          
+          echo "=== Process information ==="
+          ps aux | grep -E "(weed|test|docker)" || true
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: lakekeeper-integration-test-logs
+          path: test/s3tables/lakekeeper/test-output.log
+          retention-days: 3
+
   s3-tables-build-verification:
     name: S3 Tables Build Verification
     runs-on: ubuntu-22.04

.github/workflows/test-s3-over-https-using-awscli.yml

@@ -35,7 +35,7 @@ jobs:
           set -e
           mkdir -p /tmp/data
           ./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
-          until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
+          until curl -s http://localhost:8333/healthz > /dev/null; do sleep 1; done
 
       - name: Setup Caddy
         run: |
@@ -54,7 +54,7 @@ jobs:
       - name: Start Caddy
         run: |
           ./caddy start
-          until curl -fsS --insecure https://localhost:8443 > /dev/null; do sleep 1; done
+          until curl -fsS --insecure https://localhost:8443/healthz > /dev/null; do sleep 1; done
 
       - name: Create Bucket
         run: |

.github/workflows/volume-server-integration-tests.yml

@@ -0,0 +1,122 @@
+name: "Volume Server Integration Tests"
+
+on:
+  pull_request:
+    branches: [ master ]
+    paths:
+      - 'test/volume_server/**'
+      - 'weed/server/**'
+      - 'weed/storage/**'
+      - 'weed/pb/volume_server.proto'
+      - 'weed/pb/volume_server_pb/**'
+      - '.github/workflows/volume-server-integration-tests.yml'
+  push:
+    branches: [ master, main ]
+    paths:
+      - 'test/volume_server/**'
+      - 'weed/server/**'
+      - 'weed/storage/**'
+      - 'weed/pb/volume_server.proto'
+      - 'weed/pb/volume_server_pb/**'
+      - '.github/workflows/volume-server-integration-tests.yml'
+
+concurrency:
+  group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  GO_VERSION: '1.24'
+  TEST_TIMEOUT: '30m'
+
+jobs:
+  volume-server-integration-tests:
+    name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
+    runs-on: ubuntu-22.04
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        test-type: [grpc, http]
+        shard: [1, 2, 3]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v6
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Build SeaweedFS binary
+        run: |
+          cd weed
+          go build -o weed .
+          chmod +x weed
+          ./weed version
+
+      - name: Run volume server integration tests
+        env:
+          WEED_BINARY: ${{ github.workspace }}/weed/weed
+        run: |
+          if [ "${{ matrix.test-type }}" == "grpc" ]; then
+            if [ "${{ matrix.shard }}" == "1" ]; then
+              TEST_PATTERN="^Test[A-H]"
+            elif [ "${{ matrix.shard }}" == "2" ]; then
+              TEST_PATTERN="^Test[I-S]"
+            else
+              TEST_PATTERN="^Test[T-Z]"
+            fi
+          else
+            if [ "${{ matrix.shard }}" == "1" ]; then
+              TEST_PATTERN="^Test[A-G]"
+            elif [ "${{ matrix.shard }}" == "2" ]; then
+              TEST_PATTERN="^Test[H-R]"
+            else
+              TEST_PATTERN="^Test[S-Z]"
+            fi
+          fi
+          echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
+          go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
+
+      - name: Collect logs on failure
+        if: failure()
+        run: |
+          mkdir -p /tmp/volume-server-it-logs
+          find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/volume-server-it-logs/ \; || true
+
+      - name: Archive logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: volume-server-integration-test-logs
+          path: /tmp/volume-server-it-logs/
+          if-no-files-found: warn
+          retention-days: 7
+
+      - name: Test summary
+        if: always()
+        run: |
+          if [ "${{ matrix.test-type }}" == "grpc" ]; then
+            if [ "${{ matrix.shard }}" == "1" ]; then
+              TEST_PATTERN="^Test[A-H]"
+            elif [ "${{ matrix.shard }}" == "2" ]; then
+              TEST_PATTERN="^Test[I-S]"
+            else
+              TEST_PATTERN="^Test[T-Z]"
+            fi
+          else
+            if [ "${{ matrix.shard }}" == "1" ]; then
+              TEST_PATTERN="^Test[A-G]"
+            elif [ "${{ matrix.shard }}" == "2" ]; then
+              TEST_PATTERN="^Test[H-R]"
+            else
+              TEST_PATTERN="^Test[S-Z]"
+            fi
+          fi
+          echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
+          echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
+          echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"

go.mod

@@ -46,7 +46,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
-	github.com/klauspost/compress v1.18.3
+	github.com/klauspost/compress v1.18.4
 	github.com/klauspost/reedsolomon v1.13.0
 	github.com/kurin/blazer v0.5.3
 	github.com/linxGnu/grocksdb v1.10.7
@@ -130,7 +130,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
-	github.com/getsentry/sentry-go v0.40.0
+	github.com/getsentry/sentry-go v0.42.0
 	github.com/gin-contrib/sessions v1.0.4
 	github.com/gin-gonic/gin v1.11.0
 	github.com/go-ldap/ldap/v3 v3.4.12
@@ -142,7 +142,7 @@ require (
 	github.com/jhump/protoreflect v1.18.0
 	github.com/lib/pq v1.11.1
 	github.com/linkedin/goavro/v2 v2.14.1
-	github.com/mattn/go-sqlite3 v1.14.33
+	github.com/mattn/go-sqlite3 v1.14.34
 	github.com/minio/crc64nvme v1.1.1
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/parquet-go/parquet-go v0.26.4
@@ -150,7 +150,7 @@ require (
 	github.com/rabbitmq/amqp091-go v1.10.0
 	github.com/rclone/rclone v1.72.1
 	github.com/rdleal/intervalst v1.5.0
-	github.com/redis/go-redis/v9 v9.17.2
+	github.com/redis/go-redis/v9 v9.18.0
 	github.com/schollz/progressbar/v3 v3.19.0
 	github.com/seaweedfs/go-fuse/v2 v2.9.1
 	github.com/shirou/gopsutil/v4 v4.26.1
@@ -297,7 +297,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
 	github.com/Azure/go-ntlmssp v0.1.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect

go.sum

@@ -569,8 +569,8 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGlu
 github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 h1:sxgSqOB9CDToiaVFpxuvb5wGgGqWa3lCShcm5o0n3bE=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3/go.mod h1:XdED8i399lEVblYHTZM8eXaP07gv4Z58IL6ueMlVlrg=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
@@ -1053,8 +1053,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
 github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
 github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
 github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
-github.com/getsentry/sentry-go v0.40.0 h1:VTJMN9zbTvqDqPwheRVLcp0qcUcM+8eFivvGocAaSbo=
-github.com/getsentry/sentry-go v0.40.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
+github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8=
+github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sessions v1.0.4 h1:ha6CNdpYiTOK/hTp05miJLbpTSNfOnFg5Jm2kbcqy8U=
 github.com/gin-contrib/sessions v1.0.4/go.mod h1:ccmkrb2z6iU2osiAHZG3x3J4suJK+OU27oqzlWOqQgs=
@@ -1487,8 +1487,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
 github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
-github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
+github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
@@ -1564,8 +1564,8 @@ github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhg
 github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
 github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0=
-github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
+github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -1814,8 +1814,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
 github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
-github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
-github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
+github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
+github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
 github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
 github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
 github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.12"
+appVersion: "4.13"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.412
+version: 4.0.413

k8s/charts/seaweedfs/templates/admin/admin-service.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ printf "%s-admin" (include "seaweedfs.fullname" .) | trunc 63 | trimSuffix "-" }}
+  name: {{ include "seaweedfs.componentName" (list . "admin") }}
   namespace: {{ .Release.Namespace }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}

k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml

@@ -134,7 +134,7 @@ spec:
               {{- if .Values.worker.adminServer }}
               -admin={{ .Values.worker.adminServer }} \
               {{- else }}
-              -admin={{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }}{{ if .Values.admin.grpcPort }}.{{ .Values.admin.grpcPort }}{{ end }} \
+              -admin={{ template "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }}{{ if .Values.admin.grpcPort }}.{{ .Values.admin.grpcPort }}{{ end }} \
               {{- end }}
               -capabilities={{ .Values.worker.capabilities }} \
               -maxConcurrent={{ .Values.worker.maxConcurrent }} \

test/java/spark/docker-compose.yml

@@ -19,9 +19,6 @@ services:
       start_period: 10s
 
   seaweedfs-volume:
-    build:
-      context: ../../../docker
-      dockerfile: Dockerfile.local
     image: seaweedfs:local
     container_name: seaweedfs-spark-volume
     ports:
@@ -43,9 +40,6 @@ services:
       start_period: 10s
 
   seaweedfs-filer:
-    build:
-      context: ../../../docker
-      dockerfile: Dockerfile.local
     image: seaweedfs:local
     container_name: seaweedfs-spark-filer
     ports:

test/java/spark/pom.xml

@@ -116,7 +116,7 @@
             <dependency>
                 <groupId>org.apache.avro</groupId>
                 <artifactId>avro</artifactId>
-                <version>1.11.4</version>
+                <version>1.11.5</version>
             </dependency>
 
             <!-- Apache ZooKeeper - Fix CVEs -->

test/kafka/kafka-client-loadtest/tools/pom.xml

@@ -41,7 +41,7 @@
         <dependency>
             <groupId>org.apache.avro</groupId>
             <artifactId>avro</artifactId>
-            <version>1.11.4</version>
+            <version>1.11.5</version>
         </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>

test/s3/cors/Makefile

@@ -12,6 +12,7 @@ FILER_PORT := 8888
 TEST_TIMEOUT := 10m
 TEST_PATTERN := TestCORS
 SERVER_DIR := test-mini-data
+S3_CONFIG := s3_test_config.json
 
 # Default target
 help:
@@ -80,13 +81,15 @@ start-server: check-deps
 	@echo "🔍 DEBUG: Creating volume directory..."
 	@mkdir -p $(SERVER_DIR)
 	@echo "🔍 DEBUG: Launching SeaweedFS S3 server in background..."
-	@echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
-	@$(WEED_BINARY) mini \
+	@echo "🔍 DEBUG: Command: AWS_ACCESS_KEY_ID=some_access_key1 AWS_SECRET_ACCESS_KEY=some_secret_key1 $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
+	@env AWS_ACCESS_KEY_ID=some_access_key1 \
+	 AWS_SECRET_ACCESS_KEY=some_secret_key1 \
+	 $(WEED_BINARY) mini \
 		-dir=$(SERVER_DIR) \
 		-s3.port=$(S3_PORT) \
 		-s3.config=$(S3_CONFIG) \
 		> weed-test.log 2>&1 & \
-		echo $$! > weed-test.pid
+		echo $$! > weed-server.pid
 	
 	@echo "Waiting for S3 server to be ready..."
 	@for i in $$(seq 1 30); do \
@@ -97,7 +100,7 @@ start-server: check-deps
 		sleep 1; \
 	done; \
 	echo "S3 server failed to start"; \
-	exit 1 > weed-server.pid
+	exit 1
 	@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
 	@echo "🔍 DEBUG: Checking if PID is still running..."
 	@sleep 2

test/s3/cors/s3_test_config.json

@@ -0,0 +1,27 @@
+{
+    "identities": [
+        {
+            "name": "anonymous",
+            "actions": [
+                "Read",
+                "List"
+            ]
+        },
+        {
+            "name": "admin",
+            "credentials": [
+                {
+                    "accessKey": "some_access_key1",
+                    "secretKey": "some_secret_key1"
+                }
+            ],
+            "actions": [
+                "Admin",
+                "Read",
+                "List",
+                "Tagging",
+                "Write"
+            ]
+        }
+    ]
+}

test/s3/iam/s3_sts_credential_prefix_test.go

@@ -0,0 +1,82 @@
+package iam
+
+import (
+	"encoding/xml"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestSTSTemporaryCredentialPrefix verifies that STS temporary credentials use ASIA prefix
+// This test ensures AWS compatibility - temporary credentials should use ASIA, not AKIA
+func TestSTSTemporaryCredentialPrefix(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	// Use test credentials from environment or fall back to defaults
+	accessKey := os.Getenv("STS_TEST_ACCESS_KEY")
+	if accessKey == "" {
+		accessKey = "admin"
+	}
+	secretKey := os.Getenv("STS_TEST_SECRET_KEY")
+	if secretKey == "" {
+		secretKey = "admin"
+	}
+
+	t.Run("assume_role_returns_asia_prefix", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"AssumeRole"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/admin"},
+			"RoleSessionName": {"asia-prefix-test"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		if resp.StatusCode != http.StatusOK {
+			t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
+			t.Skip("AssumeRole not fully implemented yet")
+		}
+
+		var stsResp AssumeRoleTestResponse
+		err = xml.Unmarshal(body, &stsResp)
+		require.NoError(t, err, "Failed to parse response: %s", string(body))
+
+		creds := stsResp.Result.Credentials
+		require.NotEmpty(t, creds.AccessKeyId, "AccessKeyId should not be empty")
+
+		// Verify ASIA prefix for temporary credentials
+		assert.True(t, strings.HasPrefix(creds.AccessKeyId, "ASIA"),
+			"Temporary credentials must use ASIA prefix (not AKIA for permanent keys), got: %s", creds.AccessKeyId)
+
+		// Verify it's NOT using AKIA (permanent credentials)
+		assert.False(t, strings.HasPrefix(creds.AccessKeyId, "AKIA"),
+			"Temporary credentials must NOT use AKIA prefix (that's for permanent IAM keys), got: %s", creds.AccessKeyId)
+
+		// Verify format: ASIA + 16 hex characters = 20 chars total
+		assert.Equal(t, 20, len(creds.AccessKeyId),
+			"Access key ID should be 20 characters (ASIA + 16 hex chars), got: %s", creds.AccessKeyId)
+
+		t.Logf("✓ Temporary credentials correctly use ASIA prefix: %s", creds.AccessKeyId)
+	})
+
+	t.Run("assume_role_with_web_identity_returns_asia_prefix", func(t *testing.T) {
+		// This test would require OIDC setup, so we'll skip it for now
+		// but the same ASIA prefix validation should apply
+		t.Skip("AssumeRoleWithWebIdentity requires OIDC provider setup")
+	})
+}

test/s3/normal/iam_test.go

@@ -1,7 +1,6 @@
 package example
 
 import (
-	"os"
 	"testing"
 	"time"
 
@@ -22,10 +21,8 @@ func TestIAMOperations(t *testing.T) {
 	// Set credentials before starting cluster
 	accessKey := "testkey123"
 	secretKey := "testsecret456"
-	os.Setenv("AWS_ACCESS_KEY_ID", accessKey)
-	os.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
-	defer os.Unsetenv("AWS_ACCESS_KEY_ID")
-	defer os.Unsetenv("AWS_SECRET_ACCESS_KEY")
+	t.Setenv("AWS_ACCESS_KEY_ID", accessKey)
+	t.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
 
 	// Create and start test cluster
 	cluster, err := startMiniCluster(t)

test/s3/normal/s3_integration_test.go

@@ -146,6 +146,14 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
 		return nil, fmt.Errorf("failed to create security.toml: %v", err)
 	}
 
+	// Set environment variables for admin credentials safely for this test
+	if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
+		t.Setenv("AWS_ACCESS_KEY_ID", "admin")
+	}
+	if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
+		t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
+	}
+
 	// Start weed mini in a goroutine by calling the command directly
 	cluster.wg.Add(1)
 	go func() {

test/s3/policy/policy_test.go

@@ -216,6 +216,14 @@ enabled = true
 	err = os.WriteFile(credentialToml, []byte(credentialConfig), 0644)
 	require.NoError(t, err)
 
+	// Set environment variables for admin credentials safely for this test
+	if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
+		t.Setenv("AWS_ACCESS_KEY_ID", "admin")
+	}
+	if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
+		t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
+	}
+
 	cluster.wg.Add(1)
 	go func() {
 		defer cluster.wg.Done()

test/s3/spark/issue_8234_repro_test.go

@@ -14,7 +14,7 @@ func setupSparkIssue8234Env(t *testing.T) *TestEnvironment {
 	}
 
 	env.StartSeaweedFS(t)
-	t.Cleanup(func() { env.Cleanup() })
+	t.Cleanup(func() { env.Cleanup(t) })
 
 	createObjectBucket(t, env, "test")
 	env.startSparkContainer(t)

test/s3/spark/issue_8285_repro_test.go

@@ -0,0 +1,180 @@
+package spark
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+func TestSparkS3TemporaryDirectoryCleanupIssue8285Regression(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping Spark integration test in short mode")
+	}
+
+	env := setupSparkIssue8234Env(t)
+
+	script := `
+import pyspark.sql.functions as F
+
+target = "s3a://test/issue-8285/output"
+
+spark.conf.set("spark.hadoop.fs.s3a.committer.name", "directory")
+spark.conf.set("spark.hadoop.fs.s3a.committer.magic.enabled", "false")
+spark.conf.set("spark.hadoop.fs.s3a.committer.staging.abort.pending.uploads", "true")
+spark.conf.set("spark.hadoop.fs.s3a.committer.staging.conflict-mode", "append")
+spark.conf.set("spark.hadoop.fs.s3a.committer.staging.tmp.path", "/tmp")
+spark.conf.set("spark.hadoop.fs.s3a.directory.marker.retention", "keep")
+
+df = spark.range(0, 200).repartition(12).withColumn("value", F.col("id") * 2)
+df.write.format("parquet").mode("overwrite").save(target)
+count = spark.read.parquet(target).count()
+print("WRITE_COUNT=" + str(count))
+`
+
+	code, output := runSparkPyScript(t, env.sparkContainer, script, env.s3Port)
+	if code != 0 {
+		t.Fatalf("Spark script exited with code %d; output:\n%s", code, output)
+	}
+	if !strings.Contains(output, "WRITE_COUNT=200") {
+		t.Fatalf("expected write/read success marker in output, got:\n%s", output)
+	}
+
+	keys := listObjectKeysByPrefix(t, env, "test", "issue-8285/")
+	var temporaryKeys []string
+	for _, key := range keys {
+		if hasTemporaryPathSegment(key) {
+			temporaryKeys = append(temporaryKeys, key)
+		}
+	}
+
+	if len(temporaryKeys) > 0 {
+		t.Fatalf("issue #8285 regression detected: found lingering _temporary artifacts: %v\nall keys: %v", temporaryKeys, keys)
+	}
+
+	temporaryCandidates := []string{
+		"issue-8285/output/_temporary/",
+		"issue-8285/output/_temporary/0/",
+		"issue-8285/output/_temporary/0/_temporary/",
+	}
+	lingering := waitForObjectsToDisappear(t, env, "test", temporaryCandidates, 35*time.Second)
+	if len(lingering) > 0 {
+		t.Fatalf("issue #8285 regression detected: lingering temporary directories: %v", lingering)
+	}
+}
+
+func listObjectKeysByPrefix(t *testing.T, env *TestEnvironment, bucketName, prefix string) []string {
+	t.Helper()
+	client := newS3Client(env)
+
+	pager := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(prefix),
+	})
+
+	var keys []string
+	for pager.HasMorePages() {
+		page, err := pager.NextPage(context.Background())
+		if err != nil {
+			t.Fatalf("failed listing objects for prefix %q: %v", prefix, err)
+		}
+		for _, object := range page.Contents {
+			keys = append(keys, aws.ToString(object.Key))
+		}
+	}
+
+	return keys
+}
+
+func headObjectInfo(t *testing.T, env *TestEnvironment, bucketName, key string) (bool, string, error) {
+	t.Helper()
+
+	client := newS3Client(env)
+	output, err := client.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(key),
+	})
+	if err == nil {
+		return true, aws.ToString(output.ContentType), nil
+	}
+
+	var notFound *s3types.NotFound
+	if strings.Contains(err.Error(), "NotFound") || strings.Contains(err.Error(), "NoSuchKey") || errors.As(err, &notFound) {
+		return false, "", nil
+	}
+	return false, "", err
+}
+
+func waitForObjectsToDisappear(t *testing.T, env *TestEnvironment, bucketName string, keys []string, timeout time.Duration) []string {
+	t.Helper()
+
+	deadline := time.Now().Add(timeout)
+	pending := make(map[string]struct{}, len(keys))
+	details := make(map[string]string, len(keys))
+	for _, key := range keys {
+		pending[key] = struct{}{}
+	}
+
+	for len(pending) > 0 && time.Now().Before(deadline) {
+		for key := range pending {
+			exists, contentType, err := headObjectInfo(t, env, bucketName, key)
+			if err != nil {
+				details[key] = fmt.Sprintf("%s (head_error=%v)", key, err)
+				continue
+			}
+			if !exists {
+				delete(pending, key)
+				delete(details, key)
+				continue
+			}
+			details[key] = fmt.Sprintf("%s (exists=true, contentType=%q)", key, contentType)
+		}
+		if len(pending) > 0 {
+			time.Sleep(2 * time.Second)
+		}
+	}
+
+	if len(pending) == 0 {
+		return nil
+	}
+
+	var lingering []string
+	for _, key := range keys {
+		if _, ok := pending[key]; !ok {
+			continue
+		}
+		if detail, hasDetail := details[key]; hasDetail {
+			lingering = append(lingering, detail)
+		} else {
+			lingering = append(lingering, key)
+		}
+	}
+	return lingering
+}
+
+func newS3Client(env *TestEnvironment) *s3.Client {
+	cfg := aws.Config{
+		Region:       "us-east-1",
+		Credentials:  aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")),
+		BaseEndpoint: aws.String(fmt.Sprintf("http://localhost:%d", env.s3Port)),
+	}
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+}
+
+func hasTemporaryPathSegment(key string) bool {
+	for _, segment := range strings.Split(strings.TrimSuffix(key, "/"), "/") {
+		if segment == "_temporary" {
+			return true
+		}
+	}
+	return false
+}

test/s3/spark/setup_test.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"sync"
 	"testing"
 	"time"
@@ -53,6 +54,8 @@ type TestEnvironment struct {
 	dockerAvailable  bool
 	weedBinary       string
 	seaweedfsDataDir string
+	weedLogPath      string
+	weedLogFile      *os.File
 	masterPort       int
 	filerPort        int
 	s3Port           int
@@ -113,6 +116,15 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
 		"-s3.config", iamConfigPath,
 		"-dir", env.seaweedfsDataDir,
 	)
+	weedLogPath := filepath.Join(env.seaweedfsDataDir, "weed-mini.log")
+	weedLogFile, err := os.Create(weedLogPath)
+	if err != nil {
+		t.Fatalf("failed to create weed log file: %v", err)
+	}
+	env.weedLogPath = weedLogPath
+	env.weedLogFile = weedLogFile
+	env.masterProcess.Stdout = weedLogFile
+	env.masterProcess.Stderr = weedLogFile
 	env.masterProcess.Env = append(os.Environ(),
 		"AWS_ACCESS_KEY_ID="+env.accessKey,
 		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
@@ -160,12 +172,30 @@ func (env *TestEnvironment) startSparkContainer(t *testing.T) {
 	env.sparkContainer = container
 }
 
-func (env *TestEnvironment) Cleanup() {
+func (env *TestEnvironment) Cleanup(t *testing.T) {
 	if env.masterProcess != nil && env.masterProcess.Process != nil {
 		_ = env.masterProcess.Process.Kill()
 		_ = env.masterProcess.Wait()
 	}
 	clearMiniProcess(env.masterProcess)
+	if env.weedLogFile != nil {
+		_ = env.weedLogFile.Close()
+	}
+
+	if t.Failed() && os.Getenv("CI") != "" && env.weedLogPath != "" {
+		logData, err := os.ReadFile(env.weedLogPath)
+		if err != nil {
+			t.Logf("failed to read weed mini log file %s: %v", env.weedLogPath, err)
+		} else {
+			// Print the tail to keep CI output manageable while preserving failure context.
+			const maxTailBytes = 64 * 1024
+			start := 0
+			if len(logData) > maxTailBytes {
+				start = len(logData) - maxTailBytes
+			}
+			t.Logf("weed mini logs (tail, %d bytes):\n%s", len(logData)-start, string(logData[start:]))
+		}
+	}
 
 	if env.sparkContainer != nil {
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

test/s3tables/catalog/iceberg_catalog_test.go

@@ -43,16 +43,15 @@ func hasDocker() bool {
 	return cmd.Run() == nil
 }
 
-// getFreePort returns an available ephemeral port
-func getFreePort() (int, error) {
+// getFreePort returns an available ephemeral port and its listener
+func getFreePort() (int, net.Listener, error) {
 	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
-		return 0, err
+		return 0, nil, err
 	}
-	defer listener.Close()
 
 	addr := listener.Addr().(*net.TCPAddr)
-	return addr.Port, nil
+	return addr.Port, listener, nil
 }
 
 // NewTestEnvironment creates a new test environment
@@ -91,43 +90,67 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
 	}
 
 	// Allocate free ephemeral ports for each service
-	s3Port, err := getFreePort()
+	var listeners []net.Listener
+	defer func() {
+		for _, l := range listeners {
+			l.Close()
+		}
+	}()
+
+	var l net.Listener
+	s3Port, l, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for S3: %v", err)
 	}
-	icebergPort, err := getFreePort()
+	listeners = append(listeners, l)
+
+	icebergPort, l, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Iceberg: %v", err)
 	}
-	s3GrpcPort, err := getFreePort()
+	listeners = append(listeners, l)
+
+	s3GrpcPort, l, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for S3 gRPC: %v", err)
 	}
-	masterPort, err := getFreePort()
+	listeners = append(listeners, l)
+
+	masterPort, l, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Master: %v", err)
 	}
-	masterGrpcPort, err := getFreePort()
+	listeners = append(listeners, l)
+
+	masterGrpcPort, l, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Master gRPC: %v", err)
 	}
-	filerPort, err := getFreePort()
+	listeners = append(listeners, l)
+
+	filerPort, l, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Filer: %v", err)
 	}
-	filerGrpcPort, err := getFreePort()
+	listeners = append(listeners, l)
+
+	filerGrpcPort, l, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Filer gRPC: %v", err)
 	}
-	volumePort, err := getFreePort()
+	listeners = append(listeners, l)
+
+	volumePort, l, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Volume: %v", err)
 	}
+	listeners = append(listeners, l)
 
-	volumeGrpcPort, err := getFreePort()
+	volumeGrpcPort, l, err := getFreePort()
 	if err != nil {
 		t.Fatalf("Failed to get free port for Volume gRPC: %v", err)
 	}
+	listeners = append(listeners, l)
 
 	return &TestEnvironment{
 		seaweedDir:      seaweedDir,
@@ -488,8 +511,10 @@ func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
 	}
 	defer resp.Body.Close()
 
+	body, _ := io.ReadAll(resp.Body)
+	t.Logf("Create table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
+
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusConflict {
-		body, _ := io.ReadAll(resp.Body)
 		t.Fatalf("Failed to create table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
 	}
 	t.Logf("Created table bucket %s", bucketName)

test/s3tables/catalog/pyiceberg_test.go

@@ -56,8 +56,6 @@ func TestPyIcebergRestCatalog(t *testing.T) {
 
 	cmd := exec.Command("docker", "run", "--rm",
 		"--add-host", "host.docker.internal:host-gateway",
-		"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", "test"),
-		"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", "test"),
 		"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
 		"-v", fmt.Sprintf("%s:/app:ro", testDir),
 		"iceberg-rest-test",
@@ -78,3 +76,69 @@ func TestPyIcebergRestCatalog(t *testing.T) {
 		t.Errorf("PyIceberg test failed: %v", err)
 	}
 }
+
+// TestPyIcebergRestCatalogAuthenticated tests the Iceberg REST Catalog using PyIceberg with authentication.
+// This test uses the default admin credentials that SeaweedFS creates on startup.
+func TestPyIcebergRestCatalogAuthenticated(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	if !env.dockerAvailable {
+		t.Skip("Docker not available, skipping PyIceberg integration test")
+	}
+
+	// Use default admin credentials
+	testAccessKey := "admin"
+	testSecretKey := "admin"
+
+	// Start SeaweedFS (it will use default admin credentials from environment if set)
+	env.StartSeaweedFS(t)
+
+	// Create the test bucket first (using unauthenticated request, which works with DefaultAllow)
+	bucketName := "pyiceberg-auth-test"
+	createTableBucket(t, env, bucketName)
+
+	// Build the test working directory path
+	testDir := filepath.Join(env.seaweedDir, "test", "s3tables", "catalog")
+
+	// Run PyIceberg test using Docker with authentication
+	catalogURL := fmt.Sprintf("http://host.docker.internal:%d", env.icebergPort)
+	s3Endpoint := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
+	warehouse := fmt.Sprintf("s3://%s/", bucketName)
+
+	// Build the test image first for faster repeated runs
+	buildCmd := exec.Command("docker", "build", "-t", "iceberg-rest-test", "-f", "Dockerfile.pyiceberg", ".")
+	buildCmd.Dir = testDir
+	if out, err := buildCmd.CombinedOutput(); err != nil {
+		t.Fatalf("Failed to build test image: %v\n%s", err, string(out))
+	}
+
+	cmd := exec.Command("docker", "run", "--rm",
+		"--add-host", "host.docker.internal:host-gateway",
+		"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
+		"-v", fmt.Sprintf("%s:/app:ro", testDir),
+		"iceberg-rest-test",
+		"python3", "/app/test_rest_catalog_auth.py",
+		"--catalog-url", catalogURL,
+		"--warehouse", warehouse,
+		"--prefix", bucketName,
+		"--access-key", testAccessKey,
+		"--secret-key", testSecretKey,
+	)
+	cmd.Dir = testDir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	t.Logf("Running PyIceberg REST catalog test with authentication...")
+	t.Logf("  Catalog URL: %s", catalogURL)
+	t.Logf("  Warehouse: %s", warehouse)
+	t.Logf("  Access Key: %s", testAccessKey)
+
+	if err := cmd.Run(); err != nil {
+		t.Errorf("PyIceberg authenticated test failed: %v", err)
+	}
+}

test/s3tables/catalog/pyiceberg_test_helpers.go

@@ -0,0 +1,36 @@
+package catalog
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"testing"
+)
+
+// verifyTableBucketMetadata verifies that a table bucket was created with proper metadata
+func verifyTableBucketMetadata(t *testing.T, env *TestEnvironment, bucketName string) {
+	t.Helper()
+
+	// Use S3Tables REST API to get the bucket
+	endpoint := fmt.Sprintf("http://localhost:%d/buckets/%s", env.s3Port, bucketName)
+
+	req, err := http.NewRequest(http.MethodGet, endpoint, nil)
+	if err != nil {
+		t.Fatalf("Failed to create request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/x-amz-json-1.1")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("Failed to get table bucket %s: %v", bucketName, err)
+	}
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+	t.Logf("Get table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
+
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("Failed to get table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
+	}
+	t.Logf("Verified table bucket %s exists with metadata", bucketName)
+}

test/s3tables/catalog/test_rest_catalog.py

@@ -201,6 +201,7 @@ def main():
                     "uri": args.catalog_url,
                     "warehouse": args.warehouse,
                     "prefix": args.prefix,
+                    "s3.anonymous": "true",  # Disable AWS request signing for unauthenticated access
                 }
             )
             print(f"Successfully connected to catalog on attempt {attempt + 1}")

test/s3tables/catalog/test_rest_catalog_auth.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+"""
+Iceberg REST Catalog Compatibility Test for SeaweedFS (Authenticated)
+
+This script tests the Iceberg REST Catalog API compatibility with authentication.
+
+Usage:
+    python3 test_rest_catalog_auth.py --catalog-url http://localhost:8182 \\
+        --access-key admin --secret-key admin
+
+Requirements:
+    pip install pyiceberg[s3fs]
+"""
+
+import argparse
+import sys
+from pyiceberg.catalog import load_catalog
+from pyiceberg.schema import Schema
+from pyiceberg.types import (
+    IntegerType,
+    LongType,
+    StringType,
+    NestedField,
+)
+from pyiceberg.exceptions import (
+    NamespaceAlreadyExistsError,
+    NoSuchNamespaceError,
+    TableAlreadyExistsError,
+    NoSuchTableError,
+)
+
+
+def test_config_endpoint(catalog):
+    """Test that the catalog config endpoint returns valid configuration."""
+    print("Testing /v1/config endpoint...")
+    # The catalog is already loaded which means config endpoint worked
+    print("  /v1/config endpoint working")
+    return True
+
+
+def test_namespace_operations(catalog, prefix):
+    """Test namespace CRUD operations."""
+    print("Testing namespace operations...")
+    namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
+    
+    # List initial namespaces
+    namespaces = catalog.list_namespaces()
+    print(f"  Initial namespaces: {namespaces}")
+    
+    # Create namespace
+    try:
+        catalog.create_namespace(namespace)
+        print(f"  Created namespace: {namespace}")
+    except NamespaceAlreadyExistsError:
+        print(f"  ! Namespace already exists: {namespace}")
+    
+    # List namespaces (should include our new one)
+    namespaces = catalog.list_namespaces()
+    if namespace in namespaces:
+        print("  Namespace appears in list")
+    else:
+        print(f"  Namespace not found in list: {namespaces}")
+        return False
+    
+    # Get namespace properties
+    try:
+        props = catalog.load_namespace_properties(namespace)
+        print(f"  Loaded namespace properties: {props}")
+    except NoSuchNamespaceError:
+        print(f"  Failed to load namespace properties")
+        return False
+    
+    return True
+
+
+def test_table_operations(catalog, prefix):
+    """Test table CRUD operations."""
+    print("Testing table operations...")
+    namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
+    table_name = "auth_test_table"
+    table_id = namespace + (table_name,)
+    
+    # Define a simple schema
+    schema = Schema(
+        NestedField(field_id=1, name="id", field_type=LongType(), required=True),
+        NestedField(field_id=2, name="name", field_type=StringType(), required=False),
+        NestedField(field_id=3, name="age", field_type=IntegerType(), required=False),
+    )
+    
+    # Create table
+    try:
+        table = catalog.create_table(
+            identifier=table_id,
+            schema=schema,
+        )
+        print(f"  Created table: {table_id}")
+    except TableAlreadyExistsError:
+        print(f"  ! Table already exists: {table_id}")
+        _ = catalog.load_table(table_id)
+    
+    # List tables
+    tables = catalog.list_tables(namespace)
+    if table_name in [t[1] for t in tables]:
+        print("  Table appears in list")
+    else:
+        print(f"  Table not found in list: {tables}")
+        return False
+    
+    # Load table
+    try:
+        loaded_table = catalog.load_table(table_id)
+        print(f"  Loaded table: {loaded_table.name()}")
+        print(f"    Schema: {loaded_table.schema()}")
+        print(f"    Location: {loaded_table.location()}")
+    except NoSuchTableError:
+        print(f"  Failed to load table")
+        return False
+    
+    return True
+
+
+def test_cleanup(catalog, prefix):
+    """Test table and namespace deletion."""
+    print("Testing cleanup operations...")
+    namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
+    table_id = namespace + ("auth_test_table",)
+    
+    # Drop table
+    try:
+        catalog.drop_table(table_id)
+        print(f"  Dropped table: {table_id}")
+    except NoSuchTableError:
+        print(f"  ! Table already deleted: {table_id}")
+    
+    # Drop namespace
+    try:
+        catalog.drop_namespace(namespace)
+        print(f"  Dropped namespace: {namespace}")
+    except NoSuchNamespaceError:
+        print(f"  ! Namespace already deleted: {namespace}")
+    except Exception as e:
+        print(f"  ? Namespace drop error (may be expected): {e}")
+    
+    return True
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Test Iceberg REST Catalog with authentication")
+    parser.add_argument("--catalog-url", required=True, help="Iceberg REST Catalog URL")
+    parser.add_argument("--warehouse", default="s3://iceberg-test/", help="Warehouse location")
+    parser.add_argument("--prefix", required=True, help="Table bucket prefix")
+    parser.add_argument("--access-key", required=True, help="AWS Access Key ID")
+    parser.add_argument("--secret-key", required=True, help="AWS Secret Access Key")
+    parser.add_argument("--skip-cleanup", action="store_true", help="Skip cleanup at the end")
+    args = parser.parse_args()
+    
+    print(f"Connecting to Iceberg REST Catalog at: {args.catalog_url}")
+    print(f"Warehouse: {args.warehouse}")
+    print(f"Prefix: {args.prefix}")
+    print(f"Using authenticated access with key: {args.access_key}")
+    print()
+    
+    # Load the REST catalog with authentication
+    import time
+    max_retries = 10
+    catalog = None
+    for attempt in range(max_retries):
+        try:
+            catalog = load_catalog(
+                "rest",
+                **{
+                    "type": "rest",
+                    "uri": args.catalog_url,
+                    "warehouse": args.warehouse,
+                    "prefix": args.prefix,
+                    "s3.access-key-id": args.access_key,
+                    "s3.secret-access-key": args.secret_key,
+                }
+            )
+            print(f"Successfully connected to catalog on attempt {attempt + 1}")
+            break
+        except Exception as e:
+            if attempt < max_retries - 1:
+                print(f"  Attempt {attempt + 1} failed, retrying in 2s... ({e})")
+                time.sleep(2)
+            else:
+                print(f"  All {max_retries} attempts failed.")
+                raise e
+    
+    # Run tests
+    tests = [
+        ("Config Endpoint", lambda: test_config_endpoint(catalog)),
+        ("Namespace Operations", lambda: test_namespace_operations(catalog, args.prefix)),
+        ("Table Operations", lambda: test_table_operations(catalog, args.prefix)),
+    ]
+    
+    if not args.skip_cleanup:
+        tests.append(("Cleanup", lambda: test_cleanup(catalog, args.prefix)))
+    
+    passed = 0
+    failed = 0
+    
+    for name, test_fn in tests:
+        print(f"\n{'='*50}")
+        try:
+            if test_fn():
+                passed += 1
+                print(f"PASSED: {name}")
+            else:
+                failed += 1
+                print(f"FAILED: {name}")
+        except Exception as e:
+            failed += 1
+            print(f"ERROR in {name}: {e}")
+    
+    print(f"\n{'='*50}")
+    print(f"Results: {passed} passed, {failed} failed")
+    
+    return 0 if failed == 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

test/s3tables/catalog_risingwave/risingwave_catalog_test.go

@@ -0,0 +1,79 @@
+package catalog_risingwave
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+)
+
+func TestRisingWaveIcebergCatalog(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	if !env.dockerAvailable {
+		t.Skip("Docker not available, skipping RisingWave integration test")
+	}
+
+	t.Log(">>> Starting SeaweedFS...")
+	env.StartSeaweedFS(t)
+	t.Log(">>> SeaweedFS started.")
+
+	tableBucket := "iceberg-tables"
+	t.Logf(">>> Creating table bucket: %s", tableBucket)
+	createTableBucket(t, env, tableBucket)
+
+	t.Log(">>> Starting RisingWave...")
+	env.StartRisingWave(t)
+	t.Log(">>> RisingWave started.")
+
+	// Create Iceberg namespace
+	createIcebergNamespace(t, env, "default")
+
+	// Create a catalog in RisingWave that points to SeaweedFS Iceberg REST API
+	icebergUri := env.dockerIcebergEndpoint()
+	s3Endpoint := env.dockerS3Endpoint()
+
+	tableName := "test_table_" + randomString(6)
+	createIcebergTable(t, env, tableBucket, "default", tableName)
+
+	sourceName := "test_source_" + randomString(6)
+	createSourceSql := fmt.Sprintf(`
+CREATE SOURCE %s WITH (
+    connector = 'iceberg',
+    catalog.type = 'rest',
+    catalog.uri = '%s',
+    catalog.name = 'default',
+    database.name = 'default',
+    table.name = '%s',
+    warehouse.path = 's3://%s',
+    s3.endpoint = '%s',
+    s3.region = 'us-east-1',
+    s3.access.key = '%s',
+    s3.secret.key = '%s',
+    s3.path.style.access = 'true',
+    catalog.rest.sigv4_enabled = 'true',
+    catalog.rest.signing_region = 'us-east-1',
+    catalog.rest.signing_name = 's3'
+);`, sourceName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey)
+
+	t.Logf(">>> Creating source %s...", sourceName)
+	runRisingWaveSQL(t, env.postgresSidecar, createSourceSql)
+
+	showSourcesOutput := runRisingWaveSQL(t, env.postgresSidecar, "SHOW SOURCES;")
+	if !strings.Contains(showSourcesOutput, sourceName) {
+		t.Fatalf("Expected source %s in SHOW SOURCES output:\n%s", sourceName, showSourcesOutput)
+	}
+
+	describeOutput := runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("DESCRIBE %s;", sourceName))
+	if !strings.Contains(describeOutput, "id") || !strings.Contains(describeOutput, "name") {
+		t.Fatalf("Expected id/name columns in DESCRIBE output:\n%s", describeOutput)
+	}
+
+	runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("SELECT * FROM %s LIMIT 0;", sourceName))
+
+	t.Log(">>> RisingWave Iceberg Catalog test passed!")
+}

test/s3tables/catalog_risingwave/risingwave_dml_test.go

@@ -0,0 +1,219 @@
+package catalog_risingwave
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestRisingWaveIcebergDML(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	if !env.dockerAvailable {
+		t.Skip("Docker not available, skipping RisingWave integration test")
+	}
+
+	t.Log(">>> Starting SeaweedFS...")
+	env.StartSeaweedFS(t)
+	t.Log(">>> SeaweedFS started.")
+
+	tableBucket := "iceberg-tables"
+	t.Logf(">>> Creating table bucket: %s", tableBucket)
+	createTableBucket(t, env, tableBucket)
+
+	t.Log(">>> Starting RisingWave...")
+	env.StartRisingWave(t)
+	t.Log(">>> RisingWave started.")
+
+	// Create Iceberg namespace
+	createIcebergNamespace(t, env, "default")
+
+	icebergUri := env.dockerIcebergEndpoint()
+	s3Endpoint := env.dockerS3Endpoint()
+
+	// 1. Test INSERT (Append-only)
+	t.Run("TestInsert", func(t *testing.T) {
+		tableName := "test_insert_" + randomString(6)
+		createIcebergTable(t, env, tableBucket, "default", tableName)
+
+		rwTableName := "rw_insert_" + randomString(6)
+		runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("CREATE TABLE %s (id int, name varchar);", rwTableName))
+
+		sinkName := "test_sink_insert_" + randomString(6)
+		createSinkSql := fmt.Sprintf(`
+CREATE SINK %s FROM %s
+WITH (
+    connector = 'iceberg',
+    catalog.type = 'rest',
+    catalog.uri = '%s',
+    catalog.name = 'default',
+    database.name = 'default',
+    table.name = '%s',
+    warehouse.path = 's3://%s',
+    s3.endpoint = '%s',
+    s3.region = 'us-east-1',
+    s3.access.key = '%s',
+    s3.secret.key = '%s',
+    s3.path.style.access = 'true',
+    catalog.rest.sigv4_enabled = 'true',
+    catalog.rest.signing_region = 'us-east-1',
+    catalog.rest.signing_name = 's3',
+    type = 'append-only',
+    force_append_only = 'true'
+);`, sinkName, rwTableName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey)
+
+		t.Logf(">>> Creating sink %s...", sinkName)
+		runRisingWaveSQL(t, env.postgresSidecar, createSinkSql)
+
+		t.Log(">>> Inserting into RisingWave table...")
+		runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("INSERT INTO %s VALUES (1, 'Alice'), (2, 'Bob');", rwTableName))
+		runRisingWaveSQL(t, env.postgresSidecar, "FLUSH;")
+
+		// Verify with Source
+		sourceName := "test_source_insert_" + randomString(6)
+		createSourceSql := fmt.Sprintf(`
+CREATE SOURCE %s WITH (
+    connector = 'iceberg',
+    catalog.type = 'rest',
+    catalog.uri = '%s',
+    catalog.name = 'default',
+    database.name = 'default',
+    table.name = '%s',
+    warehouse.path = 's3://%s',
+    s3.endpoint = '%s',
+    s3.region = 'us-east-1',
+    s3.access.key = '%s',
+    s3.secret.key = '%s',
+    s3.path.style.access = 'true',
+    catalog.rest.sigv4_enabled = 'true',
+    catalog.rest.signing_region = 'us-east-1',
+    catalog.rest.signing_name = 's3'
+);`, sourceName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey)
+
+		runRisingWaveSQL(t, env.postgresSidecar, createSourceSql)
+
+		t.Log(">>> Selecting from source to verify INSERT...")
+		verifyQuery(t, env, sourceName, "1 | Alice", "2 | Bob")
+	})
+
+	// 2. Test UPSERT (Update/Delete)
+	t.Run("TestUpsert", func(t *testing.T) {
+		tableName := "test_upsert_" + randomString(6)
+		// We need a table with PK for upsert to work effectively in RW logic,
+		// effectively maps to Iceberg v2 table.
+		createIcebergTable(t, env, tableBucket, "default", tableName)
+
+		rwTableName := "rw_upsert_" + randomString(6)
+		runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("CREATE TABLE %s (id int PRIMARY KEY, name varchar);", rwTableName))
+
+		sinkName := "test_sink_upsert_" + randomString(6)
+		createSinkSql := fmt.Sprintf(`
+CREATE SINK %s FROM %s
+WITH (
+    connector = 'iceberg',
+    catalog.type = 'rest',
+    catalog.uri = '%s',
+    catalog.name = 'default',
+    database.name = 'default',
+    table.name = '%s',
+    warehouse.path = 's3://%s',
+    s3.endpoint = '%s',
+    s3.region = 'us-east-1',
+    s3.access.key = '%s',
+    s3.secret.key = '%s',
+    s3.path.style.access = 'true',
+    catalog.rest.sigv4_enabled = 'true',
+    catalog.rest.signing_region = 'us-east-1',
+    catalog.rest.signing_name = 's3',
+    type = 'upsert', -- Upsert mode
+    primary_key = 'id'
+);`, sinkName, rwTableName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey)
+
+		t.Logf(">>> Creating upsert sink %s...", sinkName)
+		runRisingWaveSQL(t, env.postgresSidecar, createSinkSql)
+
+		t.Log(">>> Inserting initial data...")
+		runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("INSERT INTO %s VALUES (1, 'Charlie'), (2, 'Dave');", rwTableName))
+		runRisingWaveSQL(t, env.postgresSidecar, "FLUSH;")
+
+		// Update 1, Delete 2
+		t.Log(">>> Updating and Deleting data...")
+		runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("UPDATE %s SET name = 'Charles' WHERE id = 1;", rwTableName))
+		runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("DELETE FROM %s WHERE id = 2;", rwTableName))
+		runRisingWaveSQL(t, env.postgresSidecar, "FLUSH;")
+
+		// Verify with Source
+		sourceName := "test_source_upsert_" + randomString(6)
+		createSourceSql := fmt.Sprintf(`
+CREATE SOURCE %s WITH (
+    connector = 'iceberg',
+    catalog.type = 'rest',
+    catalog.uri = '%s',
+    catalog.name = 'default',
+    database.name = 'default',
+    table.name = '%s',
+    warehouse.path = 's3://%s',
+    s3.endpoint = '%s',
+    s3.region = 'us-east-1',
+    s3.access.key = '%s',
+    s3.secret.key = '%s',
+    s3.path.style.access = 'true',
+    catalog.rest.sigv4_enabled = 'true',
+    catalog.rest.signing_region = 'us-east-1',
+    catalog.rest.signing_name = 's3'
+);`, sourceName, icebergUri, tableName, tableBucket, s3Endpoint, env.accessKey, env.secretKey)
+
+		runRisingWaveSQL(t, env.postgresSidecar, createSourceSql)
+
+		t.Log(">>> Selecting from source to verify UPSERT...")
+		// Should see (1, 'Charles') and NOT (2, 'Dave')
+		verifyQuery(t, env, sourceName, "1 | Charles")
+		verifyQueryAbsence(t, env, sourceName, "2 | Dave")
+	})
+}
+
+func verifyQuery(t *testing.T, env *TestEnvironment, sourceName string, expectedSubstrings ...string) {
+	t.Helper()
+	var output string
+	for i := 0; i < 15; i++ {
+		output = runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("SELECT * FROM %s ORDER BY id;", sourceName))
+		allFound := true
+		for _, s := range expectedSubstrings {
+			if !strings.Contains(output, s) {
+				allFound = false
+				break
+			}
+		}
+		if allFound {
+			return
+		}
+		time.Sleep(2 * time.Second)
+	}
+	t.Fatalf("Failed to find expected data %v in output:\n%s", expectedSubstrings, output)
+}
+
+func verifyQueryAbsence(t *testing.T, env *TestEnvironment, sourceName string, unexpectedSubstrings ...string) {
+	t.Helper()
+	var output string
+	for i := 0; i < 15; i++ {
+		output = runRisingWaveSQL(t, env.postgresSidecar, fmt.Sprintf("SELECT * FROM %s ORDER BY id;", sourceName))
+		noneFound := true
+		for _, s := range unexpectedSubstrings {
+			if strings.Contains(output, s) {
+				noneFound = false
+				break
+			}
+		}
+		if noneFound {
+			return
+		}
+		time.Sleep(2 * time.Second)
+	}
+	t.Fatalf("Found unexpected data %v in output:\n%s", unexpectedSubstrings, output)
+}

test/s3tables/catalog_risingwave/setup_test.go

@@ -0,0 +1,481 @@
+package catalog_risingwave
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"math/rand"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+)
+
+var (
+	miniProcessMu   sync.Mutex
+	lastMiniProcess *exec.Cmd
+)
+
+func stopPreviousMini() {
+	miniProcessMu.Lock()
+	defer miniProcessMu.Unlock()
+
+	if lastMiniProcess != nil && lastMiniProcess.Process != nil {
+		_ = lastMiniProcess.Process.Kill()
+		_ = lastMiniProcess.Wait()
+	}
+	lastMiniProcess = nil
+}
+
+func registerMiniProcess(cmd *exec.Cmd) {
+	miniProcessMu.Lock()
+	lastMiniProcess = cmd
+	miniProcessMu.Unlock()
+}
+
+func clearMiniProcess(cmd *exec.Cmd) {
+	miniProcessMu.Lock()
+	if lastMiniProcess == cmd {
+		lastMiniProcess = nil
+	}
+	miniProcessMu.Unlock()
+}
+
+type TestEnvironment struct {
+	t                   *testing.T
+	dockerAvailable     bool
+	seaweedfsDataDir    string
+	masterPort          int
+	filerPort           int
+	s3Port              int
+	icebergRestPort     int
+	risingwavePort      int
+	bindIP              string
+	accessKey           string
+	secretKey           string
+	risingwaveContainer string
+	postgresSidecar     string
+	masterProcess       *exec.Cmd
+	logFile             *os.File
+}
+
+func NewTestEnvironment(t *testing.T) *TestEnvironment {
+	env := &TestEnvironment{
+		t:         t,
+		accessKey: "AKIAIOSFODNN7EXAMPLE",
+		secretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+	}
+
+	// Check if Docker is available
+	cmd := exec.Command("docker", "version")
+	env.dockerAvailable = cmd.Run() == nil
+
+	return env
+}
+
+func (env *TestEnvironment) hostMasterAddress() string {
+	return fmt.Sprintf("127.0.0.1:%d", env.masterPort)
+}
+
+func (env *TestEnvironment) hostS3Endpoint() string {
+	return fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
+}
+
+func (env *TestEnvironment) hostIcebergEndpoint() string {
+	return fmt.Sprintf("http://127.0.0.1:%d", env.icebergRestPort)
+}
+
+func (env *TestEnvironment) dockerS3Endpoint() string {
+	return fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
+}
+
+func (env *TestEnvironment) dockerIcebergEndpoint() string {
+	return fmt.Sprintf("http://host.docker.internal:%d", env.icebergRestPort)
+}
+
+func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
+	t.Helper()
+
+	stopPreviousMini()
+
+	var err error
+	env.seaweedfsDataDir, err = os.MkdirTemp("", "seaweed-risingwave-test-")
+	if err != nil {
+		t.Fatalf("failed to create temp directory: %v", err)
+	}
+
+	env.masterPort = mustFreePort(t, "Master")
+	env.filerPort = mustFreePort(t, "Filer")
+	env.s3Port = mustFreePort(t, "S3")
+	env.icebergRestPort = mustFreePort(t, "Iceberg")
+	env.risingwavePort = mustFreePort(t, "RisingWave")
+
+	env.bindIP = testutil.FindBindIP()
+
+	iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)
+	if err != nil {
+		t.Fatalf("failed to create IAM config: %v", err)
+	}
+
+	// Create log file for SeaweedFS
+	logFile, err := os.Create(filepath.Join(env.seaweedfsDataDir, "seaweedfs.log"))
+	if err != nil {
+		t.Fatalf("failed to create log file: %v", err)
+	}
+	env.logFile = logFile
+
+	// Start SeaweedFS using weed mini (all-in-one including Iceberg REST)
+	env.masterProcess = exec.Command(
+		"weed", "mini",
+		"-ip", env.bindIP,
+		"-ip.bind", "0.0.0.0",
+		"-master.port", fmt.Sprintf("%d", env.masterPort),
+		"-filer.port", fmt.Sprintf("%d", env.filerPort),
+		"-s3.port", fmt.Sprintf("%d", env.s3Port),
+		"-s3.port.iceberg", fmt.Sprintf("%d", env.icebergRestPort),
+		"-s3.config", iamConfigPath,
+		"-dir", env.seaweedfsDataDir,
+	)
+	env.masterProcess.Stdout = logFile
+	env.masterProcess.Stderr = logFile
+	env.masterProcess.Env = append(os.Environ(),
+		"AWS_ACCESS_KEY_ID="+env.accessKey,
+		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
+		"ICEBERG_WAREHOUSE=s3://iceberg-tables",
+		"S3TABLES_DEFAULT_BUCKET=iceberg-tables",
+	)
+	if err := env.masterProcess.Start(); err != nil {
+		t.Fatalf("failed to start weed mini: %v", err)
+	}
+	registerMiniProcess(env.masterProcess)
+
+	// Wait for all services to be ready
+	if !waitForPort(env.masterPort, 15*time.Second) {
+		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
+	}
+	if !waitForPort(env.filerPort, 15*time.Second) {
+		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
+	}
+	if !waitForPort(env.s3Port, 15*time.Second) {
+		t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
+	}
+	if !waitForPort(env.icebergRestPort, 15*time.Second) {
+		t.Fatalf("weed mini failed to start - iceberg rest port %d not listening", env.icebergRestPort)
+	}
+}
+
+func mustFreePort(t *testing.T, name string) int {
+	t.Helper()
+	minPort := 10000
+	maxPort := 55000 // Ensure port+10000 < 65535
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	for i := 0; i < 1000; i++ {
+		port := minPort + r.Intn(maxPort-minPort)
+
+		// Check http port
+		ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+		if err != nil {
+			continue
+		}
+		ln.Close()
+
+		// Check grpc port (weed mini uses port+10000)
+		ln2, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port+10000))
+		if err != nil {
+			continue
+		}
+		ln2.Close()
+
+		return port
+	}
+	t.Fatalf("failed to find a free port < %d for %s after 1000 attempts", maxPort, name)
+	return 0
+}
+
+func waitForPort(port int, timeout time.Duration) bool {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
+		if err == nil {
+			conn.Close()
+			return true
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return false
+}
+
+func (env *TestEnvironment) StartRisingWave(t *testing.T) {
+	t.Helper()
+
+	containerName := "seaweed-risingwave-" + randomString(8)
+	env.risingwaveContainer = containerName
+
+	cmd := exec.Command("docker", "run", "-d",
+		"--name", containerName,
+		"-p", fmt.Sprintf("%d:4566", env.risingwavePort),
+		"--add-host", "host.docker.internal:host-gateway",
+		"-e", "AWS_ACCESS_KEY_ID="+env.accessKey,
+		"-e", "AWS_SECRET_ACCESS_KEY="+env.secretKey,
+		"-e", "AWS_REGION=us-east-1",
+		"-e", "AWS_S3_PATH_STYLE_ACCESS=true",
+		"-e", "AWS_S3_FORCE_PATH_STYLE=true",
+		"risingwavelabs/risingwave:v2.5.0",
+		"playground",
+	)
+	if output, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("failed to start RisingWave container: %v\n%s", err, string(output))
+	}
+
+	// Start a sidecar postgres container for running psql commands
+	sidecarName := "seaweed-risingwave-sidecar-" + randomString(8)
+	env.postgresSidecar = sidecarName
+	sidecarCmd := exec.Command("docker", "run", "-d", "--rm",
+		"--name", sidecarName,
+		"--network", fmt.Sprintf("container:%s", containerName),
+		"postgres:16-alpine",
+		"sleep", "infinity",
+	)
+	if output, err := sidecarCmd.CombinedOutput(); err != nil {
+		t.Fatalf("failed to start postgres sidecar: %v\n%s", err, string(output))
+	}
+
+	// Wait for RisingWave port to be open on host
+	if !waitForPort(env.risingwavePort, 120*time.Second) {
+		t.Fatalf("timed out waiting for RisingWave port %d to be open", env.risingwavePort)
+	}
+
+	// Wait for RisingWave to be truly ready via psql in the sidecar.
+	if !env.waitForRisingWave(120 * time.Second) {
+		t.Fatalf("timed out waiting for RisingWave to be ready via psql")
+	}
+}
+
+func (env *TestEnvironment) waitForRisingWave(timeout time.Duration) bool {
+	deadline := time.Now().Add(timeout)
+	env.t.Logf(">>> Waiting for RisingWave to be ready (timeout %v)...\n", timeout)
+	for time.Now().Before(deadline) {
+		if output, err := runPostgresClientSQL(env.postgresSidecar, "SELECT 1;"); err == nil {
+			env.t.Logf(">>> RisingWave is ready.\n")
+			return true
+		} else {
+			env.t.Logf(">>> RisingWave not ready yet: %v (Output: %s)\n", err, string(output))
+		}
+		time.Sleep(5 * time.Second)
+	}
+	return false
+}
+
+func runPostgresClientSQL(containerName, sql string) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, "docker", "exec",
+		containerName,
+		"psql",
+		"-h", "127.0.0.1",
+		"-p", "4566",
+		"-U", "root",
+		"-d", "dev",
+		"-v", "ON_ERROR_STOP=1",
+		"-c", sql,
+	)
+	return cmd.CombinedOutput()
+}
+
+func (env *TestEnvironment) Cleanup(t *testing.T) {
+	t.Helper()
+
+	if env.risingwaveContainer != "" {
+		if t.Failed() {
+			logs, err := exec.Command("docker", "logs", env.risingwaveContainer).CombinedOutput()
+			if err == nil {
+				env.t.Logf(">>> RisingWave Logs:\n%s\n", string(logs))
+			} else {
+				env.t.Logf(">>> Failed to get RisingWave logs: %v\n", err)
+			}
+		}
+		_ = exec.Command("docker", "rm", "-f", env.risingwaveContainer).Run()
+	}
+
+	if env.postgresSidecar != "" {
+		_ = exec.Command("docker", "rm", "-f", env.postgresSidecar).Run()
+	}
+
+	if env.seaweedfsDataDir != "" && t.Failed() {
+		logPath := filepath.Join(env.seaweedfsDataDir, "seaweedfs.log")
+		if content, err := os.ReadFile(logPath); err == nil {
+			env.t.Logf(">>> SeaweedFS Logs:\n%s\n", string(content))
+		}
+		env.t.Logf(">>> Filer Contents:\n")
+		listFilerContents(t, env, "/")
+	}
+
+	if env.masterProcess != nil && env.masterProcess.Process != nil {
+		_ = env.masterProcess.Process.Kill()
+		_ = env.masterProcess.Wait()
+	}
+	clearMiniProcess(env.masterProcess)
+
+	if env.seaweedfsDataDir != "" {
+		if env.logFile != nil {
+			env.logFile.Close()
+		}
+		_ = os.RemoveAll(env.seaweedfsDataDir)
+	}
+}
+
+func runRisingWaveSQL(t *testing.T, containerName, sql string) string {
+	t.Helper()
+
+	output, err := runPostgresClientSQL(containerName, sql)
+	if err != nil {
+		t.Fatalf("RisingWave command failed: %v\nSQL: %s\nOutput:\n%s", err, sql, string(output))
+	}
+	return string(output)
+}
+
+func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, "weed", "shell",
+		fmt.Sprintf("-master=%s", env.hostMasterAddress()),
+	)
+	cmd.Stdin = strings.NewReader(fmt.Sprintf("s3tables.bucket -create -name %s -account 000000000000\nexit\n", bucketName))
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("failed to create table bucket %s via weed shell: %v\nOutput: %s", bucketName, err, string(output))
+	}
+}
+
+func doIcebergSignedJSONRequest(env *TestEnvironment, method, path string, payload any) (int, string, error) {
+	url := env.hostIcebergEndpoint() + path
+
+	var body io.Reader
+	var payloadHash string
+
+	if payload != nil {
+		data, err := json.Marshal(payload)
+		if err != nil {
+			return 0, "", err
+		}
+		body = bytes.NewReader(data)
+		// hash := sha256.Sum256(data)
+		// payloadHash = hex.EncodeToString(hash[:])
+		payloadHash = "UNSIGNED-PAYLOAD"
+	} else {
+		payloadHash = "UNSIGNED-PAYLOAD"
+	}
+
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return 0, "", fmt.Errorf("failed to create request: %w", err)
+	}
+
+	if payload != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	req.Header.Set("X-Amz-Content-Sha256", payloadHash)
+
+	// Sign the request
+	credsProvider := credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")
+	creds, err := credsProvider.Retrieve(context.Background())
+	if err != nil {
+		return 0, "", fmt.Errorf("failed to retrieve credentials: %w", err)
+	}
+	signer := v4.NewSigner()
+
+	if err := signer.SignHTTP(context.Background(), creds, req, payloadHash, "s3", "us-east-1", time.Now()); err != nil {
+		return 0, "", fmt.Errorf("failed to sign request: %w", err)
+	}
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return 0, "", fmt.Errorf("request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return 0, "", fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	return resp.StatusCode, string(respBody), nil
+}
+
+func createIcebergNamespace(t *testing.T, env *TestEnvironment, namespace string) {
+	t.Helper()
+
+	status, raw, err := doIcebergSignedJSONRequest(env, "POST", "/v1/namespaces", map[string]any{
+		"namespace": []string{namespace},
+	})
+	if err != nil {
+		t.Fatalf("failed to create Iceberg namespace %s: %v", namespace, err)
+	}
+	if status != 200 && status != 409 {
+		t.Fatalf("failed to create Iceberg namespace %s: status %d body: %s", namespace, status, raw)
+	}
+}
+
+func createIcebergTable(t *testing.T, env *TestEnvironment, bucketName, namespace, tableName string) {
+	t.Helper()
+
+	createPath := fmt.Sprintf("/v1/namespaces/%s/tables", namespace)
+	status, raw, err := doIcebergSignedJSONRequest(env, "POST", createPath, map[string]any{
+		"name":     tableName,
+		"location": fmt.Sprintf("s3://%s/%s/%s", bucketName, namespace, tableName),
+		"schema": map[string]any{
+			"type": "struct",
+			"fields": []map[string]any{
+				{"id": 1, "name": "id", "required": false, "type": "int"},
+				{"id": 2, "name": "name", "required": false, "type": "string"},
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("failed to create Iceberg table %s.%s in bucket %s: %v", namespace, tableName, bucketName, err)
+	}
+	if status != 200 && status != 409 {
+		t.Fatalf("failed to create Iceberg table %s.%s in bucket %s: status %d body: %s", namespace, tableName, bucketName, status, raw)
+	}
+}
+
+func listFilerContents(t *testing.T, env *TestEnvironment, path string) {
+	t.Helper()
+
+	cmd := exec.Command("weed", "shell",
+		fmt.Sprintf("-master=%s", env.hostMasterAddress()),
+	)
+	cmd.Stdin = strings.NewReader(fmt.Sprintf("fs.ls -R %s\nexit\n", path))
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		env.t.Logf(">>> Warning: failed to list filer contents: %v\nOutput: %s\n", err, string(output))
+	} else {
+		env.t.Logf("%s\n", string(output))
+	}
+}
+
+func randomString(n int) string {
+	const letters = "abcdefghijklmnopqrstuvwxyz0123456789"
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+	return string(b)
+}

test/s3tables/lakekeeper/lakekeeper_test.go

@@ -0,0 +1,341 @@
+package lakekeeper
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+)
+
+type TestEnvironment struct {
+	seaweedDir     string
+	weedBinary     string
+	dataDir        string
+	bindIP         string
+	s3Port         int
+	s3GrpcPort     int
+	masterPort     int
+	masterGrpcPort int
+	filerPort      int
+	filerGrpcPort  int
+	volumePort     int
+	volumeGrpcPort int
+	weedProcess    *exec.Cmd
+	weedCancel     context.CancelFunc
+	accessKey      string
+	secretKey      string
+}
+
+func TestLakekeeperIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !testutil.HasDocker() {
+		t.Skip("Docker not available, skipping Lakekeeper integration test")
+	}
+
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	fmt.Printf(">>> Starting SeaweedFS with Lakekeeper configuration...\n")
+	env.StartSeaweedFS(t)
+	fmt.Printf(">>> SeaweedFS started.\n")
+
+	// Run python script in docker to test STS and S3 operations
+	runLakekeeperRepro(t, env)
+}
+
+func NewTestEnvironment(t *testing.T) *TestEnvironment {
+	t.Helper()
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("Failed to get working directory: %v", err)
+	}
+
+	seaweedDir := wd
+	for i := 0; i < 6; i++ {
+		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
+			break
+		}
+		seaweedDir = filepath.Dir(seaweedDir)
+	}
+
+	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
+	if _, err := os.Stat(weedBinary); err != nil {
+		weedBinary = "weed"
+		if _, err := exec.LookPath(weedBinary); err != nil {
+			t.Skip("weed binary not found, skipping integration test")
+		}
+	}
+
+	dataDir, err := os.MkdirTemp("", "seaweed-lakekeeper-test-*")
+	if err != nil {
+		t.Fatalf("Failed to create temp dir: %v", err)
+	}
+
+	bindIP := testutil.FindBindIP()
+
+	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
+	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
+	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
+	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
+
+	return &TestEnvironment{
+		seaweedDir:     seaweedDir,
+		weedBinary:     weedBinary,
+		dataDir:        dataDir,
+		bindIP:         bindIP,
+		s3Port:         s3Port,
+		s3GrpcPort:     s3GrpcPort,
+		masterPort:     masterPort,
+		masterGrpcPort: masterGrpcPort,
+		filerPort:      filerPort,
+		filerGrpcPort:  filerGrpcPort,
+		volumePort:     volumePort,
+		volumeGrpcPort: volumeGrpcPort,
+		accessKey:      "admin",
+		secretKey:      "admin",
+	}
+}
+
+func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
+	t.Helper()
+
+	iamConfigPath := filepath.Join(env.dataDir, "iam.json")
+	// Note: signingKey must be base64 encoded for []byte JSON unmarshaling
+	iamConfig := fmt.Sprintf(`{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        {
+          "accessKey": "%s",
+          "secretKey": "%s"
+        }
+      ],
+      "actions": ["Admin", "Read", "List", "Tagging", "Write"]
+    }
+  ],
+  "sts": {
+    "tokenDuration": "12h",
+    "maxSessionLength": "24h",
+    "issuer": "seaweedfs-sts",
+    "signingKey": "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz"
+  },
+  "roles": [
+    {
+      "roleName": "LakekeeperVendedRole",
+      "roleArn": "arn:aws:iam::000000000000:role/LakekeeperVendedRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": "*",
+            "Action": "sts:AssumeRole"
+          }
+        ]
+      },
+      "attachedPolicies": ["FullAccess"]
+    }
+  ],
+  "policies": [
+    {
+      "name": "FullAccess",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": "*",
+            "Resource": "*"
+          }
+        ]
+      }
+    }
+  ]
+}`, env.accessKey, env.secretKey)
+
+	if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
+		t.Fatalf("Failed to create IAM config: %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	env.weedCancel = cancel
+
+	// Start weed mini with both S3 config (standard IAM) and IAM config (advanced IAM/STS)
+	cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini",
+		"-master.port", fmt.Sprintf("%d", env.masterPort),
+		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
+		"-volume.port", fmt.Sprintf("%d", env.volumePort),
+		"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
+		"-filer.port", fmt.Sprintf("%d", env.filerPort),
+		"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
+		"-s3.port", fmt.Sprintf("%d", env.s3Port),
+		"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
+		"-s3.config", iamConfigPath,
+		"-s3.iam.config", iamConfigPath,
+		"-s3.iam.readOnly=false",
+		"-ip", env.bindIP,
+		"-ip.bind", "0.0.0.0",
+		"-dir", env.dataDir,
+	)
+	cmd.Dir = env.dataDir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("Failed to start SeaweedFS: %v", err)
+	}
+	env.weedProcess = cmd
+
+	if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/status", env.s3Port), 30*time.Second) {
+		t.Fatalf("S3 API failed to become ready")
+	}
+}
+
+func (env *TestEnvironment) Cleanup(t *testing.T) {
+	t.Helper()
+	if env.weedCancel != nil {
+		env.weedCancel()
+	}
+	if env.weedProcess != nil {
+		time.Sleep(1 * time.Second)
+		_ = env.weedProcess.Wait()
+	}
+	if env.dataDir != "" {
+		_ = os.RemoveAll(env.dataDir)
+	}
+}
+
+func runLakekeeperRepro(t *testing.T, env *TestEnvironment) {
+	t.Helper()
+
+	scriptContent := fmt.Sprintf(`
+import boto3
+import botocore.config
+import botocore
+from botocore.exceptions import ClientError
+import os
+import sys
+import time
+import logging
+
+# Enable botocore debug logging to see signature calculation
+logging.basicConfig(level=logging.DEBUG)
+botocore.session.get_session().set_debug_logger()
+
+print("Starting Lakekeeper repro test...")
+
+endpoint_url = "http://host.docker.internal:%d"
+access_key = "%s"
+secret_key = "%s"
+region = "us-east-1"
+
+print(f"Connecting to {endpoint_url}")
+
+try:
+    config = botocore.config.Config(
+        retries={'max_attempts': 3}
+    )
+    sts = boto3.client(
+        'sts',
+        endpoint_url=endpoint_url,
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        region_name=region,
+        config=config
+    )
+
+    role_arn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
+    session_name = "lakekeeper-session"
+
+    print(f"Calling AssumeRole on {role_arn} with POST body...")
+    
+    # Standard boto3 call sends parameters in POST body
+    response = sts.assume_role(
+        RoleArn=role_arn,
+        RoleSessionName=session_name
+    )
+
+    creds = response['Credentials']
+    access_key_id = creds['AccessKeyId']
+    secret_access_key = creds['SecretAccessKey']
+    session_token = creds['SessionToken']
+
+    print(f"Success! Got credentials with prefix: {access_key_id[:4]}")
+    
+    if not access_key_id.startswith("ASIA"):
+        print(f"FAILED: Expected ASIA prefix, got {access_key_id}")
+        sys.exit(1)
+
+    print("Verifying S3 operations with vended credentials...")
+    s3 = boto3.client(
+        's3',
+        endpoint_url=endpoint_url,
+        aws_access_key_id=access_key_id,
+        aws_secret_access_key=secret_access_key,
+        aws_session_token=session_token,
+        region_name=region,
+        config=config
+    )
+
+    bucket = "lakekeeper-vended-bucket"
+    print(f"Creating bucket {bucket}...")
+    s3.create_bucket(Bucket=bucket)
+
+    print("Listing buckets...")
+    response = s3.list_buckets()
+    buckets = [b['Name'] for b in response['Buckets']]
+    print(f"Found buckets: {buckets}")
+
+    if bucket not in buckets:
+        print(f"FAILED: Bucket {bucket} not found in list")
+        sys.exit(1)
+
+    print("SUCCESS: Lakekeeper flow verified!")
+    sys.exit(0)
+    
+except Exception as e:
+    print(f"FAILED: {e}")
+    # Print more details if it is a ClientError
+    if hasattr(e, 'response'):
+        print(f"Response: {e.response}")
+    sys.exit(1)
+`, env.s3Port, env.accessKey, env.secretKey)
+
+	scriptPath := filepath.Join(env.dataDir, "lakekeeper_repro.py")
+	if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil {
+		t.Fatalf("Failed to write python script: %v", err)
+	}
+
+	containerName := "seaweed-lakekeeper-client-" + fmt.Sprintf("%d", time.Now().UnixNano())
+
+	// Create a context with timeout for the docker run command
+	dockerCtx, dockerCancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer dockerCancel()
+
+	cmd := exec.CommandContext(dockerCtx, "docker", "run", "--rm",
+		"--name", containerName,
+		"--add-host", "host.docker.internal:host-gateway",
+		"-v", fmt.Sprintf("%s:/work", env.dataDir),
+		"python:3",
+		"/bin/bash", "-c", "pip install boto3 && python /work/lakekeeper_repro.py",
+	)
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		if dockerCtx.Err() == context.DeadlineExceeded {
+			t.Fatalf("Lakekeeper repro client timed out after 5 minutes\nOutput:\n%s", string(output))
+		}
+		t.Fatalf("Lakekeeper repro client failed: %v\nOutput:\n%s", err, string(output))
+	}
+	t.Logf("Lakekeeper repro client output:\n%s", string(output))
+}

test/s3tables/sts_integration/sts_integration_test.go

@@ -0,0 +1,409 @@
+package sts_integration
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+)
+
+// TestEnvironment mirrors the one in trino_catalog_test.go but simplified
+type TestEnvironment struct {
+	seaweedDir      string
+	weedBinary      string
+	dataDir         string
+	bindIP          string
+	s3Port          int
+	s3GrpcPort      int
+	masterPort      int
+	masterGrpcPort  int
+	filerPort       int
+	filerGrpcPort   int
+	volumePort      int
+	volumeGrpcPort  int
+	weedProcess     *exec.Cmd
+	weedCancel      context.CancelFunc
+	dockerAvailable bool
+	accessKey       string
+	secretKey       string
+}
+
+const testSTSIntegrationSigningKey = "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz" // gitleaks:allow - test-signing-key-for-sts-integration-tests
+
+func TestSTSIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	if !env.dockerAvailable {
+		t.Skip("Docker not available, skipping STS integration test")
+	}
+
+	fmt.Printf(">>> Starting SeaweedFS...\n")
+	env.StartSeaweedFS(t)
+	fmt.Printf(">>> SeaweedFS started.\n")
+
+	// Run python script in docker to test STS
+	runPythonSTSClient(t, env)
+}
+
+func NewTestEnvironment(t *testing.T) *TestEnvironment {
+	t.Helper()
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("Failed to get working directory: %v", err)
+	}
+
+	seaweedDir := wd
+	for i := 0; i < 6; i++ {
+		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
+			break
+		}
+		seaweedDir = filepath.Dir(seaweedDir)
+	}
+
+	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
+	info, err := os.Stat(weedBinary)
+	if err != nil || info.IsDir() {
+		weedBinary = "weed"
+		if _, err := exec.LookPath(weedBinary); err != nil {
+			t.Skip("weed binary not found, skipping integration test")
+		}
+	}
+
+	if !testutil.HasDocker() {
+		t.Skip("Docker not available, skipping integration test")
+	}
+
+	// Create a unique temporary directory for this test run
+	dataDir, err := os.MkdirTemp("", "seaweed-sts-test-*")
+	if err != nil {
+		t.Fatalf("Failed to create temp dir: %v", err)
+	}
+	// The Cleanup method will remove this directory, so no need for defer here.
+
+	bindIP := testutil.FindBindIP()
+
+	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
+	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
+	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
+	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3") // Changed to use testutil.MustFreePortPair
+
+	return &TestEnvironment{
+		seaweedDir:      seaweedDir,
+		weedBinary:      weedBinary,
+		dataDir:         dataDir,
+		bindIP:          bindIP,
+		s3Port:          s3Port,
+		s3GrpcPort:      s3GrpcPort,
+		masterPort:      masterPort,
+		masterGrpcPort:  masterGrpcPort,
+		filerPort:       filerPort,
+		filerGrpcPort:   filerGrpcPort,
+		volumePort:      volumePort,
+		volumeGrpcPort:  volumeGrpcPort,
+		dockerAvailable: testutil.HasDocker(),
+		accessKey:       "admin",
+		secretKey:       "adminadmin",
+	}
+}
+
+func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
+	t.Helper()
+
+	iamConfigPath := filepath.Join(env.dataDir, "iam.json")
+	// Note: signingKey must be base64 encoded for []byte JSON unmarshaling
+	iamConfig := fmt.Sprintf(`{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        { "accessKey": "%s", "secretKey": "%s" }
+      ],
+      "actions": ["Admin", "Read", "Write", "List", "Tagging"]
+    }
+  ],
+  "sts": {
+    "tokenDuration": "1h",
+    "maxSessionLength": "12h",
+    "issuer": "seaweedfs-sts",
+    "signingKey": "%s"
+  },
+  "policy": {
+    "defaultEffect": "Deny",
+    "storeType": "memory"
+  },
+  "policies": [
+    {
+      "name": "S3FullAccessPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": ["*"]
+          }
+        ]
+      }
+    }
+  ],
+  "roles": [
+    {
+      "roleName": "TestRole",
+      "roleArn": "arn:aws:iam::role/TestRole",
+      "attachedPolicies": ["S3FullAccessPolicy"],
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": "*",
+            "Action": ["sts:AssumeRole"]
+          }
+        ]
+      }
+    }
+  ]
+}`, env.accessKey, env.secretKey, testSTSIntegrationSigningKey)
+	if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
+		t.Fatalf("Failed to create IAM config: %v", err)
+	}
+
+	// Create empty security.toml
+	securityToml := filepath.Join(env.dataDir, "security.toml")
+	if err := os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644); err != nil {
+		t.Fatalf("Failed to create security.toml: %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	env.weedCancel = cancel
+
+	cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
+		"-master.port", fmt.Sprintf("%d", env.masterPort),
+		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
+		"-volume.port", fmt.Sprintf("%d", env.volumePort),
+		"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
+		"-filer.port", fmt.Sprintf("%d", env.filerPort),
+		"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
+		"-s3.port", fmt.Sprintf("%d", env.s3Port),
+		"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
+		"-s3.config", iamConfigPath,
+		"-s3.iam.config", iamConfigPath,
+		"-s3.iam.readOnly", "false",
+		"-ip", env.bindIP,
+		"-ip.bind", "0.0.0.0",
+		"-dir", env.dataDir,
+	)
+	cmd.Dir = env.dataDir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("Failed to start SeaweedFS: %v", err)
+	}
+	env.weedProcess = cmd
+
+	// Wait for S3 API to be ready
+	if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/status", env.s3Port), 30*time.Second) {
+		t.Fatalf("S3 API failed to become ready")
+	}
+}
+
+func (env *TestEnvironment) Start(t *testing.T) {
+	if !testutil.HasDocker() {
+		t.Skip("Docker not available")
+	}
+}
+
+func (env *TestEnvironment) Cleanup(t *testing.T) {
+	t.Helper()
+	if env.weedCancel != nil {
+		env.weedCancel()
+	}
+	if env.weedProcess != nil {
+		time.Sleep(1 * time.Second)
+		_ = env.weedProcess.Wait()
+	}
+	if env.dataDir != "" {
+		_ = os.RemoveAll(env.dataDir)
+	}
+}
+
+func runPythonSTSClient(t *testing.T, env *TestEnvironment) {
+	t.Helper()
+
+	// Write python script to temp dir
+	scriptContent := fmt.Sprintf(`
+import boto3
+import botocore.config
+from botocore.exceptions import ClientError
+import json
+import sys
+import time
+import urllib.error
+import urllib.request
+
+print("Starting STS inline session policy test...")
+
+primary_endpoint = "http://host.docker.internal:%d"
+fallback_endpoint = "http://%s:%d"
+access_key = "%s"
+secret_key = "%s"
+region = "us-east-1"
+
+try:
+    def wait_for_endpoint(url, timeout=30):
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            try:
+                with urllib.request.urlopen(url, timeout=2):
+                    return True
+            except urllib.error.HTTPError:
+                return True
+            except Exception:
+                time.sleep(1)
+        return False
+
+    def select_endpoint(urls):
+        for url in urls:
+            if wait_for_endpoint(url):
+                return url
+        raise Exception("No reachable S3 endpoint from container")
+
+    endpoint_url = select_endpoint([primary_endpoint, fallback_endpoint])
+    print(f"Using endpoint {endpoint_url}")
+
+    config = botocore.config.Config(
+        retries={'max_attempts': 0},
+        s3={'addressing_style': 'path'}
+    )
+    admin_s3 = boto3.client(
+        's3',
+        endpoint_url=endpoint_url,
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        region_name=region,
+        config=config
+    )
+
+    bucket = f"sts-inline-policy-{int(time.time() * 1000)}"
+    key = "allowed.txt"
+
+    print(f"Creating bucket {bucket} with admin credentials")
+    admin_s3.create_bucket(Bucket=bucket)
+    admin_s3.put_object(Bucket=bucket, Key=key, Body=b"ok")
+
+    sts = boto3.client(
+        'sts',
+        endpoint_url=endpoint_url,
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        region_name=region,
+        config=config
+    )
+
+    role_arn = "arn:aws:iam::role/TestRole"
+    session_name = "test-session"
+    session_policy = json.dumps({
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": ["s3:ListBucket"],
+                "Resource": [f"arn:aws:s3:::{bucket}"]
+            },
+            {
+                "Effect": "Allow",
+                "Action": ["s3:GetObject"],
+                "Resource": [f"arn:aws:s3:::{bucket}/*"]
+            }
+        ]
+    })
+
+    print(f"Calling AssumeRole on {role_arn} with inline session policy")
+    response = sts.assume_role(
+        RoleArn=role_arn,
+        RoleSessionName=session_name,
+        Policy=session_policy
+    )
+
+    creds = response['Credentials']
+    vended_s3 = boto3.client(
+        's3',
+        endpoint_url=endpoint_url,
+        aws_access_key_id=creds['AccessKeyId'],
+        aws_secret_access_key=creds['SecretAccessKey'],
+        aws_session_token=creds['SessionToken'],
+        region_name=region,
+        config=config
+    )
+
+    print("Listing objects (allowed)")
+    list_resp = vended_s3.list_objects_v2(Bucket=bucket)
+    keys = [obj.get('Key') for obj in list_resp.get('Contents', [])]
+    if key not in keys:
+        print(f"FAILED: Expected to see {key} in list_objects_v2 results")
+        sys.exit(1)
+
+    print("Getting object (allowed)")
+    body = vended_s3.get_object(Bucket=bucket, Key=key)['Body'].read()
+    if body != b"ok":
+        print("FAILED: Unexpected object content")
+        sys.exit(1)
+
+    print("Putting object (expected to be denied)")
+    try:
+        vended_s3.put_object(Bucket=bucket, Key="denied.txt", Body=b"no")
+        print("FAILED: PutObject unexpectedly succeeded")
+        sys.exit(1)
+    except ClientError as e:
+        error_code = e.response.get('Error', {}).get('Code', '')
+        if error_code != 'AccessDenied':
+            print(f"FAILED: Expected AccessDenied, got {error_code}")
+            sys.exit(1)
+        print("PutObject correctly denied by inline session policy")
+
+    print("SUCCESS: Inline session policy downscoping verified")
+    sys.exit(0)
+except Exception as e:
+    print(f"FAILED: {e}")
+    if hasattr(e, 'response'):
+        print(f"Response: {e.response}")
+    import traceback
+    traceback.print_exc()
+    sys.exit(1)
+`, env.s3Port, env.bindIP, env.s3Port, env.accessKey, env.secretKey)
+
+	scriptPath := filepath.Join(env.dataDir, "sts_test.py")
+	if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil {
+		t.Fatalf("Failed to write python script: %v", err)
+	}
+
+	containerName := "seaweed-sts-client-" + fmt.Sprintf("%d", time.Now().UnixNano())
+
+	cmd := exec.Command("docker", "run", "--rm",
+		"--name", containerName,
+		"--add-host", "host.docker.internal:host-gateway",
+		"-v", fmt.Sprintf("%s:/work", env.dataDir),
+		"python:3",
+		"/bin/bash", "-c", "pip install boto3 && python /work/sts_test.py",
+	)
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("Python STS client failed: %v\nOutput:\n%s", err, string(output))
+	}
+	t.Logf("Python STS client output:\n%s", string(output))
+}
+
+// Helpers copied from trino_catalog_test.go

test/s3tables/table-buckets/s3tables_integration_test.go

@@ -556,6 +556,14 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
 		return nil, fmt.Errorf("failed to create security.toml: %v", err)
 	}
 
+	// Set environment variables for admin credentials safely for this test
+	if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
+		t.Setenv("AWS_ACCESS_KEY_ID", "admin")
+	}
+	if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
+		t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
+	}
+
 	// Start weed mini in a goroutine by calling the command directly
 	cluster.wg.Add(1)
 	go func() {

test/s3tables/testutil/docker.go

@@ -0,0 +1,66 @@
+package testutil
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"os/exec"
+	"testing"
+	"time"
+)
+
+func HasDocker() bool {
+	cmd := exec.Command("docker", "version")
+	return cmd.Run() == nil
+}
+
+func MustFreePortPair(t *testing.T, name string) (int, int) {
+	httpPort, grpcPort, err := findAvailablePortPair()
+	if err != nil {
+		t.Fatalf("Failed to get free port pair for %s: %v", name, err)
+	}
+	return httpPort, grpcPort
+}
+
+func findAvailablePortPair() (int, int, error) {
+	httpPort, err := GetFreePort()
+	if err != nil {
+		return 0, 0, err
+	}
+	grpcPort, err := GetFreePort()
+	if err != nil {
+		return 0, 0, err
+	}
+	return httpPort, grpcPort, nil
+}
+
+func GetFreePort() (int, error) {
+	listener, err := net.Listen("tcp", "0.0.0.0:0")
+	if err != nil {
+		return 0, err
+	}
+	defer listener.Close()
+	return listener.Addr().(*net.TCPAddr).Port, nil
+}
+
+func WaitForService(url string, timeout time.Duration) bool {
+	client := &http.Client{Timeout: 2 * time.Second}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return false
+		case <-ticker.C:
+			resp, err := client.Get(url)
+			if err == nil {
+				resp.Body.Close()
+				return true
+			}
+		}
+	}
+}

test/volume_server/Makefile

@@ -0,0 +1,7 @@
+.PHONY: test-volume-server test-volume-server-short
+
+test-volume-server:
+	go test ./test/volume_server/... -v
+
+test-volume-server-short:
+	go test ./test/volume_server/... -short -v

test/volume_server/README.md

@@ -0,0 +1,27 @@
+# Volume Server Integration Tests
+
+This package contains integration tests for SeaweedFS volume server HTTP and gRPC APIs.
+
+## Run Tests
+
+Run tests from repo root:
+
+```bash
+go test ./test/volume_server/... -v
+```
+
+If a `weed` binary is not found, the harness will build one automatically.
+
+## Optional environment variables
+
+- `WEED_BINARY`: explicit path to the `weed` executable (disables auto-build).
+- `VOLUME_SERVER_IT_KEEP_LOGS=1`: keep temporary test directories and process logs.
+
+## Current scope (Phase 0)
+
+- Shared cluster/framework utilities
+- Matrix profile definitions
+- Initial HTTP admin endpoint checks
+- Initial gRPC state/status checks
+
+More API coverage is tracked in `/Users/chris/dev/seaweedfs2/test/volume_server/DEV_PLAN.md`.

test/volume_server/framework/cluster.go

@@ -0,0 +1,442 @@
+package framework
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+const (
+	defaultWaitTimeout    = 30 * time.Second
+	defaultWaitTick       = 200 * time.Millisecond
+	testVolumeSizeLimitMB = 32
+)
+
+// Cluster is a lightweight SeaweedFS master + one volume server test harness.
+type Cluster struct {
+	testingTB testing.TB
+	profile   matrix.Profile
+
+	weedBinary string
+	baseDir    string
+	configDir  string
+	logsDir    string
+	keepLogs   bool
+
+	masterPort     int
+	masterGrpcPort int
+	volumePort     int
+	volumeGrpcPort int
+	volumePubPort  int
+
+	masterCmd *exec.Cmd
+	volumeCmd *exec.Cmd
+
+	cleanupOnce sync.Once
+}
+
+// StartSingleVolumeCluster boots one master and one volume server.
+func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
+	t.Helper()
+
+	weedBinary, err := FindOrBuildWeedBinary()
+	if err != nil {
+		t.Fatalf("resolve weed binary: %v", err)
+	}
+
+	baseDir, keepLogs, err := newWorkDir()
+	if err != nil {
+		t.Fatalf("create temp test directory: %v", err)
+	}
+
+	configDir := filepath.Join(baseDir, "config")
+	logsDir := filepath.Join(baseDir, "logs")
+	masterDataDir := filepath.Join(baseDir, "master")
+	volumeDataDir := filepath.Join(baseDir, "volume")
+	for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir} {
+		if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
+			t.Fatalf("create %s: %v", dir, mkErr)
+		}
+	}
+
+	if err = writeSecurityConfig(configDir, profile); err != nil {
+		t.Fatalf("write security config: %v", err)
+	}
+
+	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	if err != nil {
+		t.Fatalf("allocate master port pair: %v", err)
+	}
+
+	ports, err := allocatePorts(3)
+	if err != nil {
+		t.Fatalf("allocate ports: %v", err)
+	}
+
+	c := &Cluster{
+		testingTB:      t,
+		profile:        profile,
+		weedBinary:     weedBinary,
+		baseDir:        baseDir,
+		configDir:      configDir,
+		logsDir:        logsDir,
+		keepLogs:       keepLogs,
+		masterPort:     masterPort,
+		masterGrpcPort: masterGrpcPort,
+		volumePort:     ports[0],
+		volumeGrpcPort: ports[1],
+		volumePubPort:  ports[0],
+	}
+	if profile.SplitPublicPort {
+		c.volumePubPort = ports[2]
+	}
+
+	if err = c.startMaster(masterDataDir); err != nil {
+		c.Stop()
+		t.Fatalf("start master: %v", err)
+	}
+	if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
+		masterLog := c.tailLog("master.log")
+		c.Stop()
+		t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
+	}
+
+	if err = c.startVolume(volumeDataDir); err != nil {
+		masterLog := c.tailLog("master.log")
+		c.Stop()
+		t.Fatalf("start volume: %v\nmaster log tail:\n%s", err, masterLog)
+	}
+	if err = c.waitForHTTP(c.VolumeAdminURL() + "/status"); err != nil {
+		volumeLog := c.tailLog("volume.log")
+		c.Stop()
+		t.Fatalf("wait for volume readiness: %v\nvolume log tail:\n%s", err, volumeLog)
+	}
+	if err = c.waitForTCP(c.VolumeGRPCAddress()); err != nil {
+		volumeLog := c.tailLog("volume.log")
+		c.Stop()
+		t.Fatalf("wait for volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog)
+	}
+
+	t.Cleanup(func() {
+		c.Stop()
+	})
+
+	return c
+}
+
+// Stop terminates all processes and cleans temporary files.
+func (c *Cluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		stopProcess(c.volumeCmd)
+		stopProcess(c.masterCmd)
+		if !c.keepLogs && !c.testingTB.Failed() {
+			_ = os.RemoveAll(c.baseDir)
+		} else if c.baseDir != "" {
+			c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir)
+		}
+	})
+}
+
+func (c *Cluster) startMaster(dataDir string) error {
+	logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"master",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.masterPort),
+		"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
+		"-mdir=" + dataDir,
+		"-peers=none",
+		"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
+		"-defaultReplication=000",
+	}
+
+	c.masterCmd = exec.Command(c.weedBinary, args...)
+	c.masterCmd.Dir = c.baseDir
+	c.masterCmd.Stdout = logFile
+	c.masterCmd.Stderr = logFile
+	return c.masterCmd.Start()
+}
+
+func (c *Cluster) startVolume(dataDir string) error {
+	logFile, err := os.Create(filepath.Join(c.logsDir, "volume.log"))
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"volume",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.volumePort),
+		"-port.grpc=" + strconv.Itoa(c.volumeGrpcPort),
+		"-port.public=" + strconv.Itoa(c.volumePubPort),
+		"-dir=" + dataDir,
+		"-max=16",
+		"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
+		"-readMode=" + c.profile.ReadMode,
+		"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
+		"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
+	}
+	if c.profile.InflightUploadTimeout > 0 {
+		args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
+	}
+	if c.profile.InflightDownloadTimeout > 0 {
+		args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
+	}
+
+	c.volumeCmd = exec.Command(c.weedBinary, args...)
+	c.volumeCmd.Dir = c.baseDir
+	c.volumeCmd.Stdout = logFile
+	c.volumeCmd.Stderr = logFile
+	return c.volumeCmd.Start()
+}
+
+func (c *Cluster) waitForHTTP(url string) error {
+	client := &http.Client{Timeout: 1 * time.Second}
+	deadline := time.Now().Add(defaultWaitTimeout)
+	for time.Now().Before(deadline) {
+		resp, err := client.Get(url)
+		if err == nil {
+			_, _ = io.Copy(io.Discard, resp.Body)
+			resp.Body.Close()
+			if resp.StatusCode < 500 {
+				return nil
+			}
+		}
+		time.Sleep(defaultWaitTick)
+	}
+	return fmt.Errorf("timed out waiting for %s", url)
+}
+
+func (c *Cluster) waitForTCP(addr string) error {
+	deadline := time.Now().Add(defaultWaitTimeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", addr, time.Second)
+		if err == nil {
+			_ = conn.Close()
+			return nil
+		}
+		time.Sleep(defaultWaitTick)
+	}
+	return fmt.Errorf("timed out waiting for tcp %s", addr)
+}
+
+func stopProcess(cmd *exec.Cmd) {
+	if cmd == nil || cmd.Process == nil {
+		return
+	}
+
+	_ = cmd.Process.Signal(os.Interrupt)
+	done := make(chan error, 1)
+	go func() {
+		done <- cmd.Wait()
+	}()
+
+	select {
+	case <-time.After(10 * time.Second):
+		_ = cmd.Process.Kill()
+		<-done
+	case <-done:
+	}
+}
+
+func allocatePorts(count int) ([]int, error) {
+	listeners := make([]net.Listener, 0, count)
+	ports := make([]int, 0, count)
+	for i := 0; i < count; i++ {
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			for _, ll := range listeners {
+				_ = ll.Close()
+			}
+			return nil, err
+		}
+		listeners = append(listeners, l)
+		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
+	}
+	for _, l := range listeners {
+		_ = l.Close()
+	}
+	return ports, nil
+}
+
+func allocateMasterPortPair() (int, int, error) {
+	for masterPort := 10000; masterPort <= 55535; masterPort++ {
+		masterGrpcPort := masterPort + 10000
+		l1, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterPort)))
+		if err != nil {
+			continue
+		}
+		l2, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterGrpcPort)))
+		if err != nil {
+			_ = l1.Close()
+			continue
+		}
+		_ = l2.Close()
+		_ = l1.Close()
+		return masterPort, masterGrpcPort, nil
+	}
+	return 0, 0, errors.New("unable to find available master port pair")
+}
+
+func newWorkDir() (dir string, keepLogs bool, err error) {
+	keepLogs = os.Getenv("VOLUME_SERVER_IT_KEEP_LOGS") == "1"
+	dir, err = os.MkdirTemp("", "seaweedfs_volume_server_it_")
+	return dir, keepLogs, err
+}
+
+func writeSecurityConfig(configDir string, profile matrix.Profile) error {
+	var b strings.Builder
+	if profile.EnableJWT {
+		if profile.JWTSigningKey == "" || profile.JWTReadKey == "" {
+			return errors.New("jwt profile requires both write and read keys")
+		}
+		b.WriteString("[jwt.signing]\n")
+		b.WriteString("key = \"")
+		b.WriteString(profile.JWTSigningKey)
+		b.WriteString("\"\n")
+		b.WriteString("expires_after_seconds = 60\n\n")
+
+		b.WriteString("[jwt.signing.read]\n")
+		b.WriteString("key = \"")
+		b.WriteString(profile.JWTReadKey)
+		b.WriteString("\"\n")
+		b.WriteString("expires_after_seconds = 60\n")
+	}
+	if b.Len() == 0 {
+		b.WriteString("# optional security config generated for integration tests\n")
+	}
+	return os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(b.String()), 0o644)
+}
+
+// FindOrBuildWeedBinary returns an executable weed binary, building one when needed.
+func FindOrBuildWeedBinary() (string, error) {
+	if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" {
+		if isExecutableFile(fromEnv) {
+			return fromEnv, nil
+		}
+		return "", fmt.Errorf("WEED_BINARY is set but not executable: %s", fromEnv)
+	}
+
+	repoRoot := ""
+	if _, file, _, ok := runtime.Caller(0); ok {
+		repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
+		candidate := filepath.Join(repoRoot, "weed", "weed")
+		if isExecutableFile(candidate) {
+			return candidate, nil
+		}
+	}
+
+	if repoRoot == "" {
+		return "", errors.New("unable to detect repository root")
+	}
+
+	binDir := filepath.Join(os.TempDir(), "seaweedfs_volume_server_it_bin")
+	if err := os.MkdirAll(binDir, 0o755); err != nil {
+		return "", fmt.Errorf("create binary directory %s: %w", binDir, err)
+	}
+	binPath := filepath.Join(binDir, "weed")
+	if isExecutableFile(binPath) {
+		return binPath, nil
+	}
+
+	cmd := exec.Command("go", "build", "-o", binPath, ".")
+	cmd.Dir = filepath.Join(repoRoot, "weed")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+	if err := cmd.Run(); err != nil {
+		return "", fmt.Errorf("build weed binary: %w\n%s", err, out.String())
+	}
+	if !isExecutableFile(binPath) {
+		return "", fmt.Errorf("built weed binary is not executable: %s", binPath)
+	}
+	return binPath, nil
+}
+
+func isExecutableFile(path string) bool {
+	info, err := os.Stat(path)
+	if err != nil || info.IsDir() {
+		return false
+	}
+	mode := info.Mode().Perm()
+	return mode&0o111 != 0
+}
+
+func (c *Cluster) tailLog(logName string) string {
+	f, err := os.Open(filepath.Join(c.logsDir, logName))
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	lines := make([]string, 0, 40)
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+		if len(lines) > 40 {
+			lines = lines[1:]
+		}
+	}
+	return strings.Join(lines, "\n")
+}
+
+func (c *Cluster) MasterAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
+}
+
+func (c *Cluster) VolumeAdminAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort))
+}
+
+func (c *Cluster) VolumePublicAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort))
+}
+
+func (c *Cluster) VolumeGRPCAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort))
+}
+
+// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort
+func (c *Cluster) VolumeServerAddress() string {
+	return fmt.Sprintf("%s.%d", c.VolumeAdminAddress(), c.volumeGrpcPort)
+}
+
+func (c *Cluster) MasterURL() string {
+	return "http://" + c.MasterAddress()
+}
+
+func (c *Cluster) VolumeAdminURL() string {
+	return "http://" + c.VolumeAdminAddress()
+}
+
+func (c *Cluster) VolumePublicURL() string {
+	return "http://" + c.VolumePublicAddress()
+}
+
+func (c *Cluster) BaseDir() string {
+	return c.baseDir
+}

test/volume_server/framework/cluster_dual.go

@@ -0,0 +1,293 @@
+package framework
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+type DualVolumeCluster struct {
+	testingTB testing.TB
+	profile   matrix.Profile
+
+	weedBinary string
+	baseDir    string
+	configDir  string
+	logsDir    string
+	keepLogs   bool
+
+	masterPort     int
+	masterGrpcPort int
+
+	volumePort0     int
+	volumeGrpcPort0 int
+	volumePubPort0  int
+	volumePort1     int
+	volumeGrpcPort1 int
+	volumePubPort1  int
+
+	masterCmd *exec.Cmd
+	volumeCmd0 *exec.Cmd
+	volumeCmd1 *exec.Cmd
+
+	cleanupOnce sync.Once
+}
+
+func StartDualVolumeCluster(t testing.TB, profile matrix.Profile) *DualVolumeCluster {
+	t.Helper()
+
+	weedBinary, err := FindOrBuildWeedBinary()
+	if err != nil {
+		t.Fatalf("resolve weed binary: %v", err)
+	}
+
+	baseDir, keepLogs, err := newWorkDir()
+	if err != nil {
+		t.Fatalf("create temp test directory: %v", err)
+	}
+
+	configDir := filepath.Join(baseDir, "config")
+	logsDir := filepath.Join(baseDir, "logs")
+	masterDataDir := filepath.Join(baseDir, "master")
+	volumeDataDir0 := filepath.Join(baseDir, "volume0")
+	volumeDataDir1 := filepath.Join(baseDir, "volume1")
+	for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir0, volumeDataDir1} {
+		if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
+			t.Fatalf("create %s: %v", dir, mkErr)
+		}
+	}
+
+	if err = writeSecurityConfig(configDir, profile); err != nil {
+		t.Fatalf("write security config: %v", err)
+	}
+
+	masterPort, masterGrpcPort, err := allocateMasterPortPair()
+	if err != nil {
+		t.Fatalf("allocate master port pair: %v", err)
+	}
+
+	ports, err := allocatePorts(6)
+	if err != nil {
+		t.Fatalf("allocate volume ports: %v", err)
+	}
+
+	c := &DualVolumeCluster{
+		testingTB:      t,
+		profile:        profile,
+		weedBinary:     weedBinary,
+		baseDir:        baseDir,
+		configDir:      configDir,
+		logsDir:        logsDir,
+		keepLogs:       keepLogs,
+		masterPort:     masterPort,
+		masterGrpcPort: masterGrpcPort,
+		volumePort0:    ports[0],
+		volumeGrpcPort0: ports[1],
+		volumePubPort0: ports[0],
+		volumePort1:    ports[2],
+		volumeGrpcPort1: ports[3],
+		volumePubPort1: ports[2],
+	}
+	if profile.SplitPublicPort {
+		c.volumePubPort0 = ports[4]
+		c.volumePubPort1 = ports[5]
+	}
+
+	if err = c.startMaster(masterDataDir); err != nil {
+		c.Stop()
+		t.Fatalf("start master: %v", err)
+	}
+	if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
+		masterLog := c.tailLog("master.log")
+		c.Stop()
+		t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
+	}
+
+	if err = c.startVolume(0, volumeDataDir0); err != nil {
+		masterLog := c.tailLog("master.log")
+		c.Stop()
+		t.Fatalf("start first volume server: %v\nmaster log tail:\n%s", err, masterLog)
+	}
+	if err = c.waitForHTTP(c.VolumeAdminURL(0) + "/status"); err != nil {
+		volumeLog := c.tailLog("volume0.log")
+		c.Stop()
+		t.Fatalf("wait for first volume readiness: %v\nvolume log tail:\n%s", err, volumeLog)
+	}
+	if err = c.waitForTCP(c.VolumeGRPCAddress(0)); err != nil {
+		volumeLog := c.tailLog("volume0.log")
+		c.Stop()
+		t.Fatalf("wait for first volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog)
+	}
+
+	if err = c.startVolume(1, volumeDataDir1); err != nil {
+		volumeLog := c.tailLog("volume0.log")
+		c.Stop()
+		t.Fatalf("start second volume server: %v\nfirst volume log tail:\n%s", err, volumeLog)
+	}
+	if err = c.waitForHTTP(c.VolumeAdminURL(1) + "/status"); err != nil {
+		volumeLog := c.tailLog("volume1.log")
+		c.Stop()
+		t.Fatalf("wait for second volume readiness: %v\nvolume log tail:\n%s", err, volumeLog)
+	}
+	if err = c.waitForTCP(c.VolumeGRPCAddress(1)); err != nil {
+		volumeLog := c.tailLog("volume1.log")
+		c.Stop()
+		t.Fatalf("wait for second volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog)
+	}
+
+	t.Cleanup(func() {
+		c.Stop()
+	})
+
+	return c
+}
+
+func (c *DualVolumeCluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		stopProcess(c.volumeCmd1)
+		stopProcess(c.volumeCmd0)
+		stopProcess(c.masterCmd)
+		if !c.keepLogs && !c.testingTB.Failed() {
+			_ = os.RemoveAll(c.baseDir)
+		} else if c.baseDir != "" {
+			c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir)
+		}
+	})
+}
+
+func (c *DualVolumeCluster) startMaster(dataDir string) error {
+	logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"master",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.masterPort),
+		"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
+		"-mdir=" + dataDir,
+		"-peers=none",
+		"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
+		"-defaultReplication=000",
+	}
+
+	c.masterCmd = exec.Command(c.weedBinary, args...)
+	c.masterCmd.Dir = c.baseDir
+	c.masterCmd.Stdout = logFile
+	c.masterCmd.Stderr = logFile
+	return c.masterCmd.Start()
+}
+
+func (c *DualVolumeCluster) startVolume(index int, dataDir string) error {
+	logName := fmt.Sprintf("volume%d.log", index)
+	logFile, err := os.Create(filepath.Join(c.logsDir, logName))
+	if err != nil {
+		return err
+	}
+
+	volumePort := c.volumePort0
+	volumeGrpcPort := c.volumeGrpcPort0
+	volumePubPort := c.volumePubPort0
+	if index == 1 {
+		volumePort = c.volumePort1
+		volumeGrpcPort = c.volumeGrpcPort1
+		volumePubPort = c.volumePubPort1
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"volume",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(volumePort),
+		"-port.grpc=" + strconv.Itoa(volumeGrpcPort),
+		"-port.public=" + strconv.Itoa(volumePubPort),
+		"-dir=" + dataDir,
+		"-max=16",
+		"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
+		"-readMode=" + c.profile.ReadMode,
+		"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
+		"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
+	}
+	if c.profile.InflightUploadTimeout > 0 {
+		args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
+	}
+	if c.profile.InflightDownloadTimeout > 0 {
+		args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
+	}
+
+	cmd := exec.Command(c.weedBinary, args...)
+	cmd.Dir = c.baseDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+
+	if err = cmd.Start(); err != nil {
+		return err
+	}
+	if index == 1 {
+		c.volumeCmd1 = cmd
+	} else {
+		c.volumeCmd0 = cmd
+	}
+	return nil
+}
+
+func (c *DualVolumeCluster) waitForHTTP(url string) error {
+	return (&Cluster{}).waitForHTTP(url)
+}
+
+func (c *DualVolumeCluster) waitForTCP(addr string) error {
+	return (&Cluster{}).waitForTCP(addr)
+}
+
+func (c *DualVolumeCluster) tailLog(logName string) string {
+	return (&Cluster{logsDir: c.logsDir}).tailLog(logName)
+}
+
+func (c *DualVolumeCluster) MasterAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
+}
+
+func (c *DualVolumeCluster) MasterURL() string {
+	return "http://" + c.MasterAddress()
+}
+
+func (c *DualVolumeCluster) VolumeAdminAddress(index int) string {
+	if index == 1 {
+		return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort1))
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort0))
+}
+
+func (c *DualVolumeCluster) VolumePublicAddress(index int) string {
+	if index == 1 {
+		return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort1))
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort0))
+}
+
+func (c *DualVolumeCluster) VolumeGRPCAddress(index int) string {
+	if index == 1 {
+		return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort1))
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort0))
+}
+
+func (c *DualVolumeCluster) VolumeAdminURL(index int) string {
+	return "http://" + c.VolumeAdminAddress(index)
+}
+
+func (c *DualVolumeCluster) VolumePublicURL(index int) string {
+	return "http://" + c.VolumePublicAddress(index)
+}

test/volume_server/framework/cluster_with_filer.go

@@ -0,0 +1,91 @@
+package framework
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+type ClusterWithFiler struct {
+	*Cluster
+
+	filerCmd      *exec.Cmd
+	filerPort     int
+	filerGrpcPort int
+}
+
+func StartSingleVolumeClusterWithFiler(t testing.TB, profile matrix.Profile) *ClusterWithFiler {
+	t.Helper()
+
+	baseCluster := StartSingleVolumeCluster(t, profile)
+
+	ports, err := allocatePorts(2)
+	if err != nil {
+		t.Fatalf("allocate filer ports: %v", err)
+	}
+
+	filerDataDir := filepath.Join(baseCluster.baseDir, "filer")
+	if mkErr := os.MkdirAll(filerDataDir, 0o755); mkErr != nil {
+		t.Fatalf("create filer data dir: %v", mkErr)
+	}
+
+	logFile, err := os.Create(filepath.Join(baseCluster.logsDir, "filer.log"))
+	if err != nil {
+		t.Fatalf("create filer log file: %v", err)
+	}
+
+	filerPort := ports[0]
+	filerGrpcPort := ports[1]
+	args := []string{
+		"-config_dir=" + baseCluster.configDir,
+		"filer",
+		"-master=127.0.0.1:" + strconv.Itoa(baseCluster.masterPort),
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(filerPort),
+		"-port.grpc=" + strconv.Itoa(filerGrpcPort),
+		"-defaultStoreDir=" + filerDataDir,
+	}
+
+	filerCmd := exec.Command(baseCluster.weedBinary, args...)
+	filerCmd.Dir = baseCluster.baseDir
+	filerCmd.Stdout = logFile
+	filerCmd.Stderr = logFile
+	if err = filerCmd.Start(); err != nil {
+		t.Fatalf("start filer: %v", err)
+	}
+
+	if err = baseCluster.waitForTCP(net.JoinHostPort("127.0.0.1", strconv.Itoa(filerGrpcPort))); err != nil {
+		filerLogTail := baseCluster.tailLog("filer.log")
+		stopProcess(filerCmd)
+		t.Fatalf("wait for filer grpc readiness: %v\nfiler log tail:\n%s", err, filerLogTail)
+	}
+
+	t.Cleanup(func() {
+		stopProcess(filerCmd)
+	})
+
+	return &ClusterWithFiler{
+		Cluster:       baseCluster,
+		filerCmd:      filerCmd,
+		filerPort:     filerPort,
+		filerGrpcPort: filerGrpcPort,
+	}
+}
+
+func (c *ClusterWithFiler) FilerAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerPort))
+}
+
+func (c *ClusterWithFiler) FilerGRPCAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerGrpcPort))
+}
+
+func (c *ClusterWithFiler) FilerServerAddress() string {
+	return fmt.Sprintf("%s.%d", c.FilerAddress(), c.filerGrpcPort)
+}

test/volume_server/framework/fault_injection.go

@@ -0,0 +1,8 @@
+package framework
+
+// Phase 0 placeholder for future fault injection utilities.
+//
+// Planned extensions:
+// - restart/kill selected processes
+// - temporary network isolation hooks
+// - master or peer outage helpers for proxy/replication branch coverage

test/volume_server/framework/grpc_client.go

@@ -0,0 +1,28 @@
+package framework
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func DialVolumeServer(t testing.TB, address string) (*grpc.ClientConn, volume_server_pb.VolumeServerClient) {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	conn, err := grpc.DialContext(ctx, address,
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithBlock(),
+	)
+	if err != nil {
+		t.Fatalf("dial volume grpc %s: %v", address, err)
+	}
+
+	return conn, volume_server_pb.NewVolumeServerClient(conn)
+}

test/volume_server/framework/http_client.go

@@ -0,0 +1,34 @@
+package framework
+
+import (
+	"io"
+	"net/http"
+	"testing"
+	"time"
+)
+
+func NewHTTPClient() *http.Client {
+	return &http.Client{Timeout: 10 * time.Second}
+}
+
+func DoRequest(t testing.TB, client *http.Client, req *http.Request) *http.Response {
+	t.Helper()
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("http request %s %s: %v", req.Method, req.URL.String(), err)
+	}
+	return resp
+}
+
+func ReadAllAndClose(t testing.TB, resp *http.Response) []byte {
+	t.Helper()
+	if resp == nil {
+		return nil
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("read response body: %v", err)
+	}
+	return body
+}

test/volume_server/framework/volume_fixture.go

@@ -0,0 +1,56 @@
+package framework
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+func AllocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string) {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{
+		VolumeId:    volumeID,
+		Collection:  collection,
+		Replication: "000",
+		Version:     uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("allocate volume %d: %v", volumeID, err)
+	}
+}
+
+func NewFileID(volumeID uint32, key uint64, cookie uint32) string {
+	return needle.NewFileId(needle.VolumeId(volumeID), key, cookie).String()
+}
+
+func UploadBytes(t testing.TB, client *http.Client, volumeURL, fid string, data []byte) *http.Response {
+	t.Helper()
+
+	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/%s", volumeURL, fid), bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("build upload request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/octet-stream")
+	req.Header.Set("Content-Length", fmt.Sprintf("%d", len(data)))
+	return DoRequest(t, client, req)
+}
+
+func ReadBytes(t testing.TB, client *http.Client, volumeURL, fid string) *http.Response {
+	t.Helper()
+
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", volumeURL, fid), nil)
+	if err != nil {
+		t.Fatalf("build read request: %v", err)
+	}
+	return DoRequest(t, client, req)
+}

test/volume_server/grpc/admin_extra_test.go

@@ -0,0 +1,445 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestVolumeNeedleStatusForUploadedFile(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(21)
+	const needleID = uint64(778899)
+	const cookie = uint32(0xA1B2C3D4)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	client := framework.NewHTTPClient()
+	payload := []byte("needle-status-payload")
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: volumeID,
+		NeedleId: needleID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeNeedleStatus failed: %v", err)
+	}
+	if statusResp.GetNeedleId() != needleID {
+		t.Fatalf("needle id mismatch: got %d want %d", statusResp.GetNeedleId(), needleID)
+	}
+	if statusResp.GetCookie() != cookie {
+		t.Fatalf("cookie mismatch: got %d want %d", statusResp.GetCookie(), cookie)
+	}
+	if statusResp.GetSize() == 0 {
+		t.Fatalf("expected non-zero needle size")
+	}
+}
+
+func TestVolumeNeedleStatusViaEcShardsWhenNormalVolumeUnmounted(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(26)
+	const needleID = uint64(778900)
+	const cookie = uint32(0xA1B2C3D5)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	payload := []byte("needle-status-ec-path-payload")
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount data shards failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeUnmount failed: %v", err)
+	}
+
+	statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: volumeID,
+		NeedleId: needleID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeNeedleStatus via EC shards failed: %v", err)
+	}
+	if statusResp.GetNeedleId() != needleID {
+		t.Fatalf("needle id mismatch: got %d want %d", statusResp.GetNeedleId(), needleID)
+	}
+	if statusResp.GetCookie() != cookie {
+		t.Fatalf("cookie mismatch: got %d want %d", statusResp.GetCookie(), cookie)
+	}
+	if statusResp.GetSize() == 0 {
+		t.Fatalf("expected non-zero needle size from EC-backed needle status")
+	}
+
+	_, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: volumeID,
+		NeedleId: needleID + 999999,
+	})
+	if err == nil || !strings.Contains(strings.ToLower(err.Error()), "not found") {
+		t.Fatalf("VolumeNeedleStatus via EC shards missing-needle error mismatch: %v", err)
+	}
+}
+
+func TestVolumeNeedleStatusMissingVolumeAndNeedle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(25)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: 99925,
+		NeedleId: 1,
+	})
+	if err == nil {
+		t.Fatalf("VolumeNeedleStatus should fail for missing volume")
+	}
+	if !strings.Contains(strings.ToLower(err.Error()), "volume not found") {
+		t.Fatalf("VolumeNeedleStatus missing-volume error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: volumeID,
+		NeedleId: 123456789,
+	})
+	if err == nil {
+		t.Fatalf("VolumeNeedleStatus should fail for missing needle")
+	}
+	if !strings.Contains(strings.ToLower(err.Error()), "not found") {
+		t.Fatalf("VolumeNeedleStatus missing-needle error mismatch: %v", err)
+	}
+}
+
+func mustNewRequest(t testing.TB, method, url string) *http.Request {
+	t.Helper()
+	req, err := http.NewRequest(method, url, nil)
+	if err != nil {
+		t.Fatalf("create request %s %s: %v", method, url, err)
+	}
+	return req
+}
+
+func TestVolumeConfigureInvalidReplication(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(22)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{
+		VolumeId:    volumeID,
+		Replication: "bad-replication",
+	})
+	if err != nil {
+		t.Fatalf("VolumeConfigure returned grpc error: %v", err)
+	}
+	if resp.GetError() == "" {
+		t.Fatalf("VolumeConfigure expected response error for invalid replication")
+	}
+	if !strings.Contains(strings.ToLower(resp.GetError()), "replication") {
+		t.Fatalf("VolumeConfigure error should mention replication, got: %q", resp.GetError())
+	}
+}
+
+func TestVolumeConfigureSuccessAndMissingRollbackPath(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(24)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	successResp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{
+		VolumeId:    volumeID,
+		Replication: "000",
+	})
+	if err != nil {
+		t.Fatalf("VolumeConfigure success path returned grpc error: %v", err)
+	}
+	if successResp.GetError() != "" {
+		t.Fatalf("VolumeConfigure success path expected empty response error, got: %q", successResp.GetError())
+	}
+
+	statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus after successful configure failed: %v", err)
+	}
+	if statusResp.GetIsReadOnly() {
+		t.Fatalf("VolumeStatus after configure expected writable volume")
+	}
+
+	missingResp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{
+		VolumeId:    99024,
+		Replication: "000",
+	})
+	if err != nil {
+		t.Fatalf("VolumeConfigure missing-volume branch should return response error, got grpc error: %v", err)
+	}
+	if missingResp.GetError() == "" {
+		t.Fatalf("VolumeConfigure missing-volume expected non-empty response error")
+	}
+	lower := strings.ToLower(missingResp.GetError())
+	if !strings.Contains(lower, "not found on disk") {
+		t.Fatalf("VolumeConfigure missing-volume error should mention not found on disk, got: %q", missingResp.GetError())
+	}
+	if !strings.Contains(lower, "failed to restore mount") {
+		t.Fatalf("VolumeConfigure missing-volume error should include remount rollback failure, got: %q", missingResp.GetError())
+	}
+}
+
+func TestPingVolumeTargetAndLeaveAffectsHealthz(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	pingResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
+		TargetType: cluster.VolumeServerType,
+		Target:     clusterHarness.VolumeServerAddress(),
+	})
+	if err != nil {
+		t.Fatalf("Ping target volume server failed: %v", err)
+	}
+	if pingResp.GetRemoteTimeNs() == 0 {
+		t.Fatalf("expected remote timestamp from ping target volume server")
+	}
+
+	if _, err = grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil {
+		t.Fatalf("VolumeServerLeave failed: %v", err)
+	}
+
+	client := framework.NewHTTPClient()
+	healthURL := clusterHarness.VolumeAdminURL() + "/healthz"
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, healthURL))
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode == http.StatusServiceUnavailable {
+			return
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("expected healthz to return 503 after leave, got %d", resp.StatusCode)
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+func TestVolumeServerLeaveIsIdempotent(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	if _, err := grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil {
+		t.Fatalf("first VolumeServerLeave failed: %v", err)
+	}
+	if _, err := grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil {
+		t.Fatalf("second VolumeServerLeave should be idempotent success, got: %v", err)
+	}
+
+	client := framework.NewHTTPClient()
+	healthURL := clusterHarness.VolumeAdminURL() + "/healthz"
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, healthURL))
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode == http.StatusServiceUnavailable {
+			return
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("expected healthz to stay 503 after repeated leave, got %d", resp.StatusCode)
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+func TestPingUnknownAndUnreachableTargetPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	unknownResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
+		TargetType: "unknown-type",
+		Target:     "127.0.0.1:12345",
+	})
+	if err != nil {
+		t.Fatalf("Ping unknown target type should not return grpc error, got: %v", err)
+	}
+	if unknownResp.GetRemoteTimeNs() != 0 {
+		t.Fatalf("Ping unknown target type expected remote_time_ns=0, got %d", unknownResp.GetRemoteTimeNs())
+	}
+	if unknownResp.GetStopTimeNs() < unknownResp.GetStartTimeNs() {
+		t.Fatalf("Ping unknown target type expected stop_time_ns >= start_time_ns")
+	}
+
+	_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
+		TargetType: cluster.MasterType,
+		Target:     "127.0.0.1:1",
+	})
+	if err == nil {
+		t.Fatalf("Ping master target should fail when target is unreachable")
+	}
+	if !strings.Contains(err.Error(), "ping master") {
+		t.Fatalf("Ping master unreachable error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
+		TargetType: cluster.FilerType,
+		Target:     "127.0.0.1:1",
+	})
+	if err == nil {
+		t.Fatalf("Ping filer target should fail when target is unreachable")
+	}
+	if !strings.Contains(err.Error(), "ping filer") {
+		t.Fatalf("Ping filer unreachable error mismatch: %v", err)
+	}
+}
+
+func TestPingMasterTargetSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
+		TargetType: cluster.MasterType,
+		Target:     clusterHarness.MasterAddress(),
+	})
+	if err != nil {
+		t.Fatalf("Ping master target success path failed: %v", err)
+	}
+	if resp.GetRemoteTimeNs() == 0 {
+		t.Fatalf("Ping master target expected non-zero remote time")
+	}
+	if resp.GetStopTimeNs() < resp.GetStartTimeNs() {
+		t.Fatalf("Ping master target expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs())
+	}
+}
+
+func TestPingFilerTargetSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeClusterWithFiler(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
+		TargetType: cluster.FilerType,
+		Target:     clusterHarness.FilerServerAddress(),
+	})
+	if err != nil {
+		t.Fatalf("Ping filer target success path failed: %v", err)
+	}
+	if resp.GetRemoteTimeNs() == 0 {
+		t.Fatalf("Ping filer target expected non-zero remote time")
+	}
+	if resp.GetStopTimeNs() < resp.GetStartTimeNs() {
+		t.Fatalf("Ping filer target expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs())
+	}
+}

test/volume_server/grpc/admin_lifecycle_test.go

@@ -0,0 +1,215 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func TestVolumeAdminLifecycleRPCs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	const volumeID = uint32(11)
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	statusResp, err := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus failed: %v", err)
+	}
+	if statusResp.GetFileCount() != 0 {
+		t.Fatalf("new volume should be empty, got file_count=%d", statusResp.GetFileCount())
+	}
+
+	if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil {
+		t.Fatalf("VolumeUnmount failed: %v", err)
+	}
+	if _, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID}); err != nil {
+		t.Fatalf("VolumeMount failed: %v", err)
+	}
+
+	if _, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}); err != nil {
+		t.Fatalf("VolumeDelete failed: %v", err)
+	}
+
+	_, err = client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err == nil {
+		t.Fatalf("VolumeStatus should fail after delete")
+	}
+	if st, ok := status.FromError(err); !ok || st.Code() == codes.OK {
+		t.Fatalf("VolumeStatus error should be a non-OK grpc status, got: %v", err)
+	}
+}
+
+func TestVolumeDeleteOnlyEmptyVariants(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(13)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 66001, 0x11223344)
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("volume-delete-only-empty"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true})
+	if err == nil || !strings.Contains(err.Error(), "volume not empty") {
+		t.Fatalf("VolumeDelete only_empty=true expected volume-not-empty error, got: %v", err)
+	}
+
+	_, err = grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: false})
+	if err != nil {
+		t.Fatalf("VolumeDelete only_empty=false failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err == nil {
+		t.Fatalf("VolumeStatus should fail after non-empty delete with only_empty=false")
+	}
+}
+
+func TestMaintenanceModeRejectsAllocateVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	_, err = client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{VolumeId: 12, Replication: "000"})
+	if err == nil {
+		t.Fatalf("AllocateVolume should fail when maintenance mode is enabled")
+	}
+	if !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("expected maintenance mode error, got: %v", err)
+	}
+}
+
+func TestAllocateDuplicateAndMountUnmountMissingVariants(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	const missingVolumeID = uint32(99331)
+	const volumeID = uint32(14)
+
+	if _, err := client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: missingVolumeID}); err != nil {
+		t.Fatalf("VolumeUnmount missing volume should be idempotent success, got: %v", err)
+	}
+
+	_, err := client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: missingVolumeID})
+	if err == nil {
+		t.Fatalf("VolumeMount missing volume should fail")
+	}
+	if !strings.Contains(err.Error(), "not found on disk") {
+		t.Fatalf("VolumeMount missing volume error mismatch: %v", err)
+	}
+
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	_, err = client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{
+		VolumeId:    volumeID,
+		Replication: "000",
+	})
+	if err == nil {
+		t.Fatalf("AllocateVolume duplicate should fail")
+	}
+	if !strings.Contains(strings.ToLower(err.Error()), "already exists") {
+		t.Fatalf("AllocateVolume duplicate error mismatch: %v", err)
+	}
+
+	if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil {
+		t.Fatalf("VolumeUnmount existing volume failed: %v", err)
+	}
+	if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil {
+		t.Fatalf("VolumeUnmount already-unmounted volume should be idempotent success, got: %v", err)
+	}
+	if _, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID}); err != nil {
+		t.Fatalf("VolumeMount remount failed: %v", err)
+	}
+}
+
+func TestMaintenanceModeRejectsVolumeDelete(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(15)
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	_, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true})
+	if err == nil {
+		t.Fatalf("VolumeDelete should fail when maintenance mode is enabled")
+	}
+	if !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("expected maintenance mode error, got: %v", err)
+	}
+}

test/volume_server/grpc/admin_readonly_collection_test.go

@@ -0,0 +1,177 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestVolumeMarkReadonlyAndWritableLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(72)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
+		VolumeId: volumeID,
+		Persist:  false,
+	})
+	if err != nil {
+		t.Fatalf("VolumeMarkReadonly failed: %v", err)
+	}
+
+	readOnlyStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus after readonly failed: %v", err)
+	}
+	if !readOnlyStatus.GetIsReadOnly() {
+		t.Fatalf("VolumeStatus expected readonly=true after VolumeMarkReadonly")
+	}
+
+	_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeMarkWritable failed: %v", err)
+	}
+
+	writableStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus after writable failed: %v", err)
+	}
+	if writableStatus.GetIsReadOnly() {
+		t.Fatalf("VolumeStatus expected readonly=false after VolumeMarkWritable")
+	}
+}
+
+func TestVolumeMarkReadonlyPersistTrue(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(74)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
+		VolumeId: volumeID,
+		Persist:  true,
+	})
+	if err != nil {
+		t.Fatalf("VolumeMarkReadonly persist=true failed: %v", err)
+	}
+
+	statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus after persist readonly failed: %v", err)
+	}
+	if !statusResp.GetIsReadOnly() {
+		t.Fatalf("VolumeStatus expected readonly=true after persist readonly")
+	}
+}
+
+func TestVolumeMarkReadonlyWritableErrorPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 98771, Persist: true})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeMarkReadonly missing-volume error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 98772})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeMarkWritable missing-volume error mismatch: %v", err)
+	}
+
+	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{
+			Maintenance: true,
+			Version:     stateResp.GetState().GetVersion(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 1, Persist: true})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeMarkReadonly maintenance error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 1})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeMarkWritable maintenance error mismatch: %v", err)
+	}
+}
+
+func TestDeleteCollectionRemovesVolumeAndIsIdempotent(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(73)
+	const collection = "it-delete-collection"
+
+	framework.AllocateVolume(t, grpcClient, volumeID, collection)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus before DeleteCollection failed: %v", err)
+	}
+
+	_, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection})
+	if err != nil {
+		t.Fatalf("DeleteCollection existing collection failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err == nil {
+		t.Fatalf("VolumeStatus should fail after collection delete")
+	}
+	if !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("VolumeStatus after DeleteCollection error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection})
+	if err != nil {
+		t.Fatalf("DeleteCollection idempotent retry failed: %v", err)
+	}
+}

test/volume_server/grpc/batch_delete_test.go

@@ -0,0 +1,264 @@
+package volume_server_grpc_test
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestBatchDeleteInvalidFidAndMaintenanceMode(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{"bad-fid"}})
+	if err != nil {
+		t.Fatalf("BatchDelete invalid fid should return response, got error: %v", err)
+	}
+	if len(resp.GetResults()) != 1 {
+		t.Fatalf("expected one batch delete result, got %d", len(resp.GetResults()))
+	}
+	if got := resp.GetResults()[0].GetStatus(); got != 400 {
+		t.Fatalf("invalid fid expected status 400, got %d", got)
+	}
+
+	stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	_, err = client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{"1,1234567890ab"}})
+	if err == nil {
+		t.Fatalf("BatchDelete should fail when maintenance mode is enabled")
+	}
+	if !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("expected maintenance mode error, got: %v", err)
+	}
+}
+
+func TestBatchDeleteCookieMismatchAndSkipCheck(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(31)
+	const needleID = uint64(900001)
+	const correctCookie = uint32(0x1122AABB)
+	const wrongCookie = uint32(0x1122AABC)
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, correctCookie)
+	uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), fid, []byte("batch-delete-cookie-check"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	wrongCookieFid := framework.NewFileID(volumeID, needleID, wrongCookie)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	mismatchResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
+		FileIds:         []string{wrongCookieFid},
+		SkipCookieCheck: false,
+	})
+	if err != nil {
+		t.Fatalf("BatchDelete with cookie check failed: %v", err)
+	}
+	if len(mismatchResp.GetResults()) != 1 {
+		t.Fatalf("BatchDelete cookie mismatch expected 1 result, got %d", len(mismatchResp.GetResults()))
+	}
+	if mismatchResp.GetResults()[0].GetStatus() != http.StatusBadRequest {
+		t.Fatalf("BatchDelete cookie mismatch expected status 400, got %d", mismatchResp.GetResults()[0].GetStatus())
+	}
+
+	skipCheckResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
+		FileIds:         []string{wrongCookieFid},
+		SkipCookieCheck: true,
+	})
+	if err != nil {
+		t.Fatalf("BatchDelete skip cookie check failed: %v", err)
+	}
+	if len(skipCheckResp.GetResults()) != 1 {
+		t.Fatalf("BatchDelete skip check expected 1 result, got %d", len(skipCheckResp.GetResults()))
+	}
+	if skipCheckResp.GetResults()[0].GetStatus() != http.StatusAccepted {
+		t.Fatalf("BatchDelete skip check expected status 202, got %d", skipCheckResp.GetResults()[0].GetStatus())
+	}
+
+	readAfterDelete := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid)
+	_ = framework.ReadAllAndClose(t, readAfterDelete)
+	if readAfterDelete.StatusCode != http.StatusNotFound {
+		t.Fatalf("read after skip-check batch delete expected 404, got %d", readAfterDelete.StatusCode)
+	}
+}
+
+func TestBatchDeleteMixedStatusesAndMismatchStopsProcessing(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(32)
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	const needleA = uint64(910001)
+	const needleB = uint64(910002)
+	const needleC = uint64(910003)
+	const cookieA = uint32(0x11111111)
+	const cookieB = uint32(0x22222222)
+	const cookieC = uint32(0x33333333)
+
+	httpClient := framework.NewHTTPClient()
+	fidA := framework.NewFileID(volumeID, needleA, cookieA)
+	fidB := framework.NewFileID(volumeID, needleB, cookieB)
+	fidC := framework.NewFileID(volumeID, needleC, cookieC)
+
+	for _, tc := range []struct {
+		fid  string
+		body string
+	}{
+		{fid: fidA, body: "batch-delete-mixed-a"},
+		{fid: fidB, body: "batch-delete-mixed-b"},
+		{fid: fidC, body: "batch-delete-mixed-c"},
+	} {
+		uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, []byte(tc.body))
+		_ = framework.ReadAllAndClose(t, uploadResp)
+		if uploadResp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s expected 201, got %d", tc.fid, uploadResp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	missingFid := framework.NewFileID(volumeID, 919999, 0x44444444)
+	mixedResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
+		FileIds: []string{"bad-fid", fidA, missingFid},
+	})
+	if err != nil {
+		t.Fatalf("BatchDelete mixed status request failed: %v", err)
+	}
+	if len(mixedResp.GetResults()) != 3 {
+		t.Fatalf("BatchDelete mixed status expected 3 results, got %d", len(mixedResp.GetResults()))
+	}
+	if mixedResp.GetResults()[0].GetStatus() != http.StatusBadRequest {
+		t.Fatalf("BatchDelete mixed result[0] expected 400, got %d", mixedResp.GetResults()[0].GetStatus())
+	}
+	if mixedResp.GetResults()[1].GetStatus() != http.StatusAccepted {
+		t.Fatalf("BatchDelete mixed result[1] expected 202, got %d", mixedResp.GetResults()[1].GetStatus())
+	}
+	if mixedResp.GetResults()[2].GetStatus() != http.StatusNotFound {
+		t.Fatalf("BatchDelete mixed result[2] expected 404, got %d", mixedResp.GetResults()[2].GetStatus())
+	}
+
+	readDeletedA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA)
+	_ = framework.ReadAllAndClose(t, readDeletedA)
+	if readDeletedA.StatusCode != http.StatusNotFound {
+		t.Fatalf("fidA should be deleted after batch delete, got status %d", readDeletedA.StatusCode)
+	}
+
+	wrongCookieB := framework.NewFileID(volumeID, needleB, cookieB+1)
+	stopResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
+		FileIds: []string{wrongCookieB, fidC},
+	})
+	if err != nil {
+		t.Fatalf("BatchDelete mismatch-stop request failed: %v", err)
+	}
+	if len(stopResp.GetResults()) != 1 {
+		t.Fatalf("BatchDelete mismatch-stop expected 1 result due early break, got %d", len(stopResp.GetResults()))
+	}
+	if stopResp.GetResults()[0].GetStatus() != http.StatusBadRequest {
+		t.Fatalf("BatchDelete mismatch-stop expected 400, got %d", stopResp.GetResults()[0].GetStatus())
+	}
+
+	readB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB)
+	_ = framework.ReadAllAndClose(t, readB)
+	if readB.StatusCode != http.StatusOK {
+		t.Fatalf("fidB should remain after cookie mismatch path, got %d", readB.StatusCode)
+	}
+
+	readC := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidC)
+	_ = framework.ReadAllAndClose(t, readC)
+	if readC.StatusCode != http.StatusOK {
+		t.Fatalf("fidC should remain when batch processing stops on mismatch, got %d", readC.StatusCode)
+	}
+}
+
+func TestBatchDeleteRejectsChunkManifestNeedles(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(33)
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 920001, 0x5555AAAA)
+	req, err := http.NewRequest(http.MethodPost, cluster.VolumeAdminURL()+"/"+fid+"?cm=true", bytes.NewReader([]byte("manifest-placeholder-payload")))
+	if err != nil {
+		t.Fatalf("create chunk manifest upload request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/octet-stream")
+	uploadResp := framework.DoRequest(t, httpClient, req)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("chunk manifest upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{fid}})
+	if err != nil {
+		t.Fatalf("BatchDelete chunk manifest should return response, got grpc error: %v", err)
+	}
+	if len(resp.GetResults()) != 1 {
+		t.Fatalf("BatchDelete chunk manifest expected one result, got %d", len(resp.GetResults()))
+	}
+	if resp.GetResults()[0].GetStatus() != http.StatusNotAcceptable {
+		t.Fatalf("BatchDelete chunk manifest expected status 406, got %d", resp.GetResults()[0].GetStatus())
+	}
+	if !strings.Contains(resp.GetResults()[0].GetError(), "ChunkManifest") {
+		t.Fatalf("BatchDelete chunk manifest expected error mentioning ChunkManifest, got %q", resp.GetResults()[0].GetError())
+	}
+
+	readResp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid)
+	_ = framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusOK {
+		t.Fatalf("chunk manifest should not be deleted by BatchDelete reject path, got %d", readResp.StatusCode)
+	}
+}

test/volume_server/grpc/copy_receive_variants_test.go

@@ -0,0 +1,431 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"io"
+	"math"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestVolumeIncrementalCopyDataAndNoDataPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(91)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 770001, 0x1122AABB)
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("incremental-copy-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	dataStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{
+		VolumeId: volumeID,
+		SinceNs:  0,
+	})
+	if err != nil {
+		t.Fatalf("VolumeIncrementalCopy start failed: %v", err)
+	}
+
+	totalBytes := 0
+	for {
+		msg, recvErr := dataStream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("VolumeIncrementalCopy recv failed: %v", recvErr)
+		}
+		totalBytes += len(msg.GetFileContent())
+	}
+	if totalBytes == 0 {
+		t.Fatalf("VolumeIncrementalCopy expected streamed bytes for since_ns=0")
+	}
+
+	noDataStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{
+		VolumeId: volumeID,
+		SinceNs:  math.MaxUint64,
+	})
+	if err != nil {
+		t.Fatalf("VolumeIncrementalCopy no-data start failed: %v", err)
+	}
+	_, err = noDataStream.Recv()
+	if err != io.EOF {
+		t.Fatalf("VolumeIncrementalCopy no-data expected EOF, got: %v", err)
+	}
+}
+
+func TestCopyFileIgnoreNotFoundAndStopOffsetZeroPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(92)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	missingNoIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:                 volumeID,
+		Ext:                      ".definitely-missing",
+		CompactionRevision:       math.MaxUint32,
+		StopOffset:               1,
+		IgnoreSourceFileNotFound: false,
+	})
+	if err == nil {
+		_, err = missingNoIgnore.Recv()
+	}
+	if err == nil {
+		t.Fatalf("CopyFile should fail for missing source file when ignore_source_file_not_found=false")
+	}
+
+	missingIgnored, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:                 volumeID,
+		Ext:                      ".definitely-missing",
+		CompactionRevision:       math.MaxUint32,
+		StopOffset:               1,
+		IgnoreSourceFileNotFound: true,
+	})
+	if err != nil {
+		t.Fatalf("CopyFile ignore-not-found start failed: %v", err)
+	}
+	_, err = missingIgnored.Recv()
+	if err != io.EOF {
+		t.Fatalf("CopyFile ignore-not-found expected EOF, got: %v", err)
+	}
+
+	stopZeroStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:                 volumeID,
+		Ext:                      ".definitely-missing",
+		CompactionRevision:       math.MaxUint32,
+		StopOffset:               0,
+		IgnoreSourceFileNotFound: false,
+	})
+	if err != nil {
+		t.Fatalf("CopyFile stop_offset=0 start failed: %v", err)
+	}
+	_, err = stopZeroStream.Recv()
+	if err != io.EOF {
+		t.Fatalf("CopyFile stop_offset=0 expected EOF, got: %v", err)
+	}
+}
+
+func TestCopyFileCompactionRevisionMismatch(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(94)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Ext:                ".idx",
+		CompactionRevision: 1, // fresh volume starts at revision 0
+		StopOffset:         1,
+	})
+	if err == nil {
+		_, err = stream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "is compacted") {
+		t.Fatalf("CopyFile compaction mismatch error mismatch: %v", err)
+	}
+}
+
+func TestReceiveFileProtocolViolationResponses(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	contentFirstStream, err := grpcClient.ReceiveFile(ctx)
+	if err != nil {
+		t.Fatalf("ReceiveFile stream create failed: %v", err)
+	}
+	if err = contentFirstStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_FileContent{
+			FileContent: []byte("content-before-info"),
+		},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send content-first failed: %v", err)
+	}
+	contentFirstResp, err := contentFirstStream.CloseAndRecv()
+	if err != nil {
+		t.Fatalf("ReceiveFile content-first close failed: %v", err)
+	}
+	if !strings.Contains(contentFirstResp.GetError(), "file info must be sent first") {
+		t.Fatalf("ReceiveFile content-first response mismatch: %+v", contentFirstResp)
+	}
+
+	unknownTypeStream, err := grpcClient.ReceiveFile(ctx)
+	if err != nil {
+		t.Fatalf("ReceiveFile stream create for unknown-type failed: %v", err)
+	}
+	if err = unknownTypeStream.Send(&volume_server_pb.ReceiveFileRequest{}); err != nil {
+		t.Fatalf("ReceiveFile send unknown-type request failed: %v", err)
+	}
+	unknownTypeResp, err := unknownTypeStream.CloseAndRecv()
+	if err != nil {
+		t.Fatalf("ReceiveFile unknown-type close failed: %v", err)
+	}
+	if !strings.Contains(unknownTypeResp.GetError(), "unknown message type") {
+		t.Fatalf("ReceiveFile unknown-type response mismatch: %+v", unknownTypeResp)
+	}
+}
+
+func TestReceiveFileSuccessForRegularVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(95)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	payloadA := []byte("receive-file-chunk-a:")
+	payloadB := []byte("receive-file-chunk-b")
+	expected := append(append([]byte{}, payloadA...), payloadB...)
+
+	receiveStream, err := grpcClient.ReceiveFile(ctx)
+	if err != nil {
+		t.Fatalf("ReceiveFile stream create failed: %v", err)
+	}
+
+	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_Info{
+			Info: &volume_server_pb.ReceiveFileInfo{
+				VolumeId:   volumeID,
+				Ext:        ".tmprecv",
+				Collection: "",
+				IsEcVolume: false,
+				FileSize:   uint64(len(expected)),
+			},
+		},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send info failed: %v", err)
+	}
+	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadA},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send payloadA failed: %v", err)
+	}
+	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadB},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send payloadB failed: %v", err)
+	}
+
+	resp, err := receiveStream.CloseAndRecv()
+	if err != nil {
+		t.Fatalf("ReceiveFile close failed: %v", err)
+	}
+	if resp.GetError() != "" {
+		t.Fatalf("ReceiveFile unexpected error response: %+v", resp)
+	}
+	if resp.GetBytesWritten() != uint64(len(expected)) {
+		t.Fatalf("ReceiveFile bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(expected))
+	}
+
+	copyStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Ext:                ".tmprecv",
+		CompactionRevision: math.MaxUint32,
+		StopOffset:         uint64(len(expected)),
+	})
+	if err != nil {
+		t.Fatalf("CopyFile for received data start failed: %v", err)
+	}
+
+	var copied []byte
+	for {
+		msg, recvErr := copyStream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("CopyFile for received data recv failed: %v", recvErr)
+		}
+		copied = append(copied, msg.GetFileContent()...)
+	}
+
+	if string(copied) != string(expected) {
+		t.Fatalf("received file data mismatch: got %q want %q", string(copied), string(expected))
+	}
+}
+
+func TestReceiveFileSuccessForEcVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	const volumeID = uint32(96)
+	const collection = "ec-receive-success"
+	const ext = ".ec00"
+
+	payloadA := []byte("receive-ec-file-chunk-a:")
+	payloadB := []byte("receive-ec-file-chunk-b")
+	expected := append(append([]byte{}, payloadA...), payloadB...)
+
+	receiveStream, err := grpcClient.ReceiveFile(ctx)
+	if err != nil {
+		t.Fatalf("ReceiveFile stream create failed: %v", err)
+	}
+
+	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_Info{
+			Info: &volume_server_pb.ReceiveFileInfo{
+				VolumeId:   volumeID,
+				Ext:        ext,
+				Collection: collection,
+				IsEcVolume: true,
+				ShardId:    0,
+				FileSize:   uint64(len(expected)),
+			},
+		},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send EC info failed: %v", err)
+	}
+	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadA},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send EC payloadA failed: %v", err)
+	}
+	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadB},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send EC payloadB failed: %v", err)
+	}
+
+	resp, err := receiveStream.CloseAndRecv()
+	if err != nil {
+		t.Fatalf("ReceiveFile EC close failed: %v", err)
+	}
+	if resp.GetError() != "" {
+		t.Fatalf("ReceiveFile EC unexpected error response: %+v", resp)
+	}
+	if resp.GetBytesWritten() != uint64(len(expected)) {
+		t.Fatalf("ReceiveFile EC bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(expected))
+	}
+
+	copyStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Collection:         collection,
+		IsEcVolume:         true,
+		Ext:                ext,
+		CompactionRevision: math.MaxUint32,
+		StopOffset:         uint64(len(expected)),
+	})
+	if err != nil {
+		t.Fatalf("CopyFile for received EC data start failed: %v", err)
+	}
+
+	var copied []byte
+	for {
+		msg, recvErr := copyStream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("CopyFile for received EC data recv failed: %v", recvErr)
+		}
+		copied = append(copied, msg.GetFileContent()...)
+	}
+
+	if string(copied) != string(expected) {
+		t.Fatalf("received EC file data mismatch: got %q want %q", string(copied), string(expected))
+	}
+}
+
+func TestCopyFileEcVolumeIgnoreMissingSourcePaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	streamNoIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:                 99601,
+		Collection:               "ec-copy-missing",
+		IsEcVolume:               true,
+		Ext:                      ".ec00",
+		CompactionRevision:       math.MaxUint32,
+		StopOffset:               1,
+		IgnoreSourceFileNotFound: false,
+	})
+	if err == nil {
+		_, err = streamNoIgnore.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found ec volume id") {
+		t.Fatalf("CopyFile EC missing source error mismatch: %v", err)
+	}
+
+	streamIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:                 99602,
+		Collection:               "ec-copy-missing",
+		IsEcVolume:               true,
+		Ext:                      ".ec00",
+		CompactionRevision:       math.MaxUint32,
+		StopOffset:               1,
+		IgnoreSourceFileNotFound: true,
+	})
+	if err != nil {
+		t.Fatalf("CopyFile EC ignore-missing start failed: %v", err)
+	}
+	_, err = streamIgnore.Recv()
+	if err != io.EOF {
+		t.Fatalf("CopyFile EC ignore-missing expected EOF, got: %v", err)
+	}
+}

test/volume_server/grpc/copy_sync_test.go

@@ -0,0 +1,284 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestVolumeSyncStatusAndReadVolumeFileStatus(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(41)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	syncResp, err := grpcClient.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeSyncStatus failed: %v", err)
+	}
+	if syncResp.GetVolumeId() != volumeID {
+		t.Fatalf("VolumeSyncStatus volume id mismatch: got %d want %d", syncResp.GetVolumeId(), volumeID)
+	}
+
+	statusResp, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus failed: %v", err)
+	}
+	if statusResp.GetVolumeId() != volumeID {
+		t.Fatalf("ReadVolumeFileStatus volume id mismatch: got %d want %d", statusResp.GetVolumeId(), volumeID)
+	}
+	if statusResp.GetVersion() == 0 {
+		t.Fatalf("ReadVolumeFileStatus expected non-zero version")
+	}
+}
+
+func TestCopyAndStreamMethodsMissingVolumePaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{VolumeId: 98761})
+	if err == nil {
+		t.Fatalf("VolumeSyncStatus should fail for missing volume")
+	}
+
+	incrementalStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{VolumeId: 98762, SinceNs: 0})
+	if err == nil {
+		_, err = incrementalStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("VolumeIncrementalCopy missing-volume error mismatch: %v", err)
+	}
+
+	readAllStream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{98763}})
+	if err == nil {
+		_, err = readAllStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("ReadAllNeedles missing-volume error mismatch: %v", err)
+	}
+
+	copyFileStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{VolumeId: 98764, Ext: ".dat", StopOffset: 1})
+	if err == nil {
+		_, err = copyFileStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("CopyFile missing-volume error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: 98765})
+	if err == nil || !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("ReadVolumeFileStatus missing-volume error mismatch: %v", err)
+	}
+}
+
+func TestVolumeCopyAndReceiveFileMaintenanceRejection(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	copyStream, err := grpcClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{VolumeId: 1, SourceDataNode: "127.0.0.1:1234"})
+	if err == nil {
+		_, err = copyStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeCopy maintenance error mismatch: %v", err)
+	}
+
+	receiveClient, err := grpcClient.ReceiveFile(ctx)
+	if err != nil {
+		t.Fatalf("ReceiveFile client creation failed: %v", err)
+	}
+	_ = receiveClient.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_Info{
+			Info: &volume_server_pb.ReceiveFileInfo{VolumeId: 1, Ext: ".dat"},
+		},
+	})
+	_, err = receiveClient.CloseAndRecv()
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("ReceiveFile maintenance error mismatch: %v", err)
+	}
+}
+
+func TestVolumeCopySuccessFromPeerAndMountsDestination(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
+	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer sourceConn.Close()
+	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
+	defer destConn.Close()
+
+	const volumeID = uint32(42)
+	framework.AllocateVolume(t, sourceClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 880001, 0x12345678)
+	payload := []byte("volume-copy-success-payload")
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload to source expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1],
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy start failed: %v", err)
+	}
+
+	sawFinalAppendTimestamp := false
+	for {
+		msg, recvErr := copyStream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("VolumeCopy recv failed: %v", recvErr)
+		}
+		if msg.GetLastAppendAtNs() > 0 {
+			sawFinalAppendTimestamp = true
+		}
+	}
+	if !sawFinalAppendTimestamp {
+		t.Fatalf("VolumeCopy expected final response with last_append_at_ns")
+	}
+
+	destReadResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid)
+	destReadBody := framework.ReadAllAndClose(t, destReadResp)
+	if destReadResp.StatusCode != http.StatusOK {
+		t.Fatalf("read from copied destination expected 200, got %d", destReadResp.StatusCode)
+	}
+	if string(destReadBody) != string(payload) {
+		t.Fatalf("destination copied payload mismatch: got %q want %q", string(destReadBody), string(payload))
+	}
+}
+
+func TestVolumeCopyOverwritesExistingDestinationVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
+	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer sourceConn.Close()
+	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
+	defer destConn.Close()
+
+	const volumeID = uint32(43)
+	framework.AllocateVolume(t, sourceClient, volumeID, "")
+	framework.AllocateVolume(t, destClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 880002, 0x23456789)
+	sourcePayload := []byte("volume-copy-overwrite-source")
+	destPayload := []byte("volume-copy-overwrite-destination-old")
+
+	sourceUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, sourcePayload)
+	_ = framework.ReadAllAndClose(t, sourceUploadResp)
+	if sourceUploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload to source expected 201, got %d", sourceUploadResp.StatusCode)
+	}
+
+	destUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid, destPayload)
+	_ = framework.ReadAllAndClose(t, destUploadResp)
+	if destUploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload to destination expected 201, got %d", destUploadResp.StatusCode)
+	}
+
+	destReadBeforeResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid)
+	destReadBeforeBody := framework.ReadAllAndClose(t, destReadBeforeResp)
+	if destReadBeforeResp.StatusCode != http.StatusOK {
+		t.Fatalf("destination pre-copy read expected 200, got %d", destReadBeforeResp.StatusCode)
+	}
+	if string(destReadBeforeBody) != string(destPayload) {
+		t.Fatalf("destination pre-copy payload mismatch: got %q want %q", string(destReadBeforeBody), string(destPayload))
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1],
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy overwrite start failed: %v", err)
+	}
+
+	sawFinalAppendTimestamp := false
+	for {
+		msg, recvErr := copyStream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("VolumeCopy overwrite recv failed: %v", recvErr)
+		}
+		if msg.GetLastAppendAtNs() > 0 {
+			sawFinalAppendTimestamp = true
+		}
+	}
+	if !sawFinalAppendTimestamp {
+		t.Fatalf("VolumeCopy overwrite expected final response with last_append_at_ns")
+	}
+
+	destReadAfterResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid)
+	destReadAfterBody := framework.ReadAllAndClose(t, destReadAfterResp)
+	if destReadAfterResp.StatusCode != http.StatusOK {
+		t.Fatalf("destination post-copy read expected 200, got %d", destReadAfterResp.StatusCode)
+	}
+	if string(destReadAfterBody) != string(sourcePayload) {
+		t.Fatalf("destination post-copy payload mismatch: got %q want %q", string(destReadAfterBody), string(sourcePayload))
+	}
+}

test/volume_server/grpc/data_rw_test.go

@@ -0,0 +1,146 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestReadNeedleBlobAndMetaMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{
+		VolumeId: 99111,
+		Offset:   0,
+		Size:     16,
+	})
+	if err == nil {
+		t.Fatalf("ReadNeedleBlob should fail for missing volume")
+	}
+	if !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("ReadNeedleBlob missing volume error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
+		VolumeId: 99112,
+		NeedleId: 1,
+		Offset:   0,
+		Size:     16,
+	})
+	if err == nil {
+		t.Fatalf("ReadNeedleMeta should fail for missing volume")
+	}
+	if !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("ReadNeedleMeta missing volume error mismatch: %v", err)
+	}
+}
+
+func TestWriteNeedleBlobMaintenanceAndMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
+		VolumeId:   99113,
+		NeedleId:   1,
+		NeedleBlob: []byte("abc"),
+		Size:       3,
+	})
+	if err == nil {
+		t.Fatalf("WriteNeedleBlob should fail for missing volume")
+	}
+	if !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("WriteNeedleBlob missing volume error mismatch: %v", err)
+	}
+
+	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
+		VolumeId:   1,
+		NeedleId:   2,
+		NeedleBlob: []byte("def"),
+		Size:       3,
+	})
+	if err == nil {
+		t.Fatalf("WriteNeedleBlob should fail in maintenance mode")
+	}
+	if !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("WriteNeedleBlob maintenance mode error mismatch: %v", err)
+	}
+}
+
+func TestReadNeedleBlobAndMetaInvalidOffsets(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(92)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 880001, 0xCCDD1122)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("invalid-offset-check"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{
+		VolumeId: volumeID,
+		Offset:   1 << 40,
+		Size:     64,
+	})
+	if err == nil {
+		t.Fatalf("ReadNeedleBlob should fail for invalid offset")
+	}
+	if !strings.Contains(strings.ToLower(err.Error()), "read needle blob") {
+		t.Fatalf("ReadNeedleBlob invalid offset error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
+		VolumeId: volumeID,
+		NeedleId: 880001,
+		Offset:   1 << 40,
+		Size:     64,
+	})
+	if err == nil {
+		t.Fatalf("ReadNeedleMeta should fail for invalid offset")
+	}
+}

test/volume_server/grpc/data_stream_success_test.go

@@ -0,0 +1,273 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(83)
+	const sourceNeedleID = uint64(333333)
+	const sourceCookie = uint32(0xABCD0102)
+	const clonedNeedleID = uint64(333334)
+
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	payload := []byte("blob-roundtrip-content")
+	fid := framework.NewFileID(volumeID, sourceNeedleID, sourceCookie)
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	fileStatus, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus failed: %v", err)
+	}
+	if fileStatus.GetIdxFileSize() == 0 {
+		t.Fatalf("expected non-zero idx file size after upload")
+	}
+
+	idxBytes := copyFileBytes(t, grpcClient, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Ext:                ".idx",
+		CompactionRevision: fileStatus.GetCompactionRevision(),
+		StopOffset:         fileStatus.GetIdxFileSize(),
+	})
+	offset, size := findNeedleOffsetAndSize(t, idxBytes, sourceNeedleID)
+
+	blobResp, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{
+		VolumeId: volumeID,
+		Offset:   offset,
+		Size:     size,
+	})
+	if err != nil {
+		t.Fatalf("ReadNeedleBlob failed: %v", err)
+	}
+	if len(blobResp.GetNeedleBlob()) == 0 {
+		t.Fatalf("ReadNeedleBlob returned empty blob")
+	}
+
+	metaResp, err := grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
+		VolumeId: volumeID,
+		NeedleId: sourceNeedleID,
+		Offset:   offset,
+		Size:     size,
+	})
+	if err != nil {
+		t.Fatalf("ReadNeedleMeta failed: %v", err)
+	}
+	if metaResp.GetCookie() != sourceCookie {
+		t.Fatalf("ReadNeedleMeta cookie mismatch: got %d want %d", metaResp.GetCookie(), sourceCookie)
+	}
+
+	_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
+		VolumeId:   volumeID,
+		NeedleId:   clonedNeedleID,
+		Size:       size,
+		NeedleBlob: blobResp.GetNeedleBlob(),
+	})
+	if err != nil {
+		t.Fatalf("WriteNeedleBlob failed: %v", err)
+	}
+
+	clonedStatus, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: volumeID,
+		NeedleId: clonedNeedleID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeNeedleStatus for cloned needle failed: %v", err)
+	}
+	if clonedStatus.GetNeedleId() != sourceNeedleID {
+		t.Fatalf("cloned needle status id mismatch: got %d want %d", clonedStatus.GetNeedleId(), sourceNeedleID)
+	}
+	if clonedStatus.GetCookie() != sourceCookie {
+		t.Fatalf("cloned needle cookie mismatch: got %d want %d", clonedStatus.GetCookie(), sourceCookie)
+	}
+
+	clonedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, clonedNeedleID, sourceCookie))
+	clonedReadBody := framework.ReadAllAndClose(t, clonedReadResp)
+	if clonedReadResp.StatusCode != 200 {
+		t.Fatalf("cloned needle GET expected 200, got %d", clonedReadResp.StatusCode)
+	}
+	if string(clonedReadBody) != string(payload) {
+		t.Fatalf("cloned needle body mismatch: got %q want %q", string(clonedReadBody), string(payload))
+	}
+}
+
+func TestReadAllNeedlesStreamsUploadedRecords(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(84)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	expected := map[uint64]string{
+		444441: "read-all-needle-one",
+		444442: "read-all-needle-two",
+	}
+	for key, body := range expected {
+		resp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, key, 0xA0B0C0D0), []byte(body))
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != 201 {
+			t.Fatalf("upload for key %d expected 201, got %d", key, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{volumeID}})
+	if err != nil {
+		t.Fatalf("ReadAllNeedles start failed: %v", err)
+	}
+
+	seen := map[uint64]string{}
+	for {
+		msg, recvErr := stream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("ReadAllNeedles recv failed: %v", recvErr)
+		}
+		if _, wanted := expected[msg.GetNeedleId()]; wanted {
+			seen[msg.GetNeedleId()] = string(msg.GetNeedleBlob())
+		}
+	}
+
+	for key, body := range expected {
+		got, found := seen[key]
+		if !found {
+			t.Fatalf("ReadAllNeedles missing key %d in stream", key)
+		}
+		if got != body {
+			t.Fatalf("ReadAllNeedles body mismatch for key %d: got %q want %q", key, got, body)
+		}
+	}
+}
+
+func TestReadAllNeedlesExistingThenMissingVolumeAbortsStream(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const existingVolumeID = uint32(85)
+	const missingVolumeID = uint32(98585)
+	const needleID = uint64(445551)
+	framework.AllocateVolume(t, grpcClient, existingVolumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(existingVolumeID, needleID, 0xAA11BB22)
+	payload := "read-all-existing-then-missing"
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte(payload))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{
+		VolumeIds: []uint32{existingVolumeID, missingVolumeID},
+	})
+	if err != nil {
+		t.Fatalf("ReadAllNeedles start failed: %v", err)
+	}
+
+	seenUploadedNeedle := false
+	for {
+		msg, recvErr := stream.Recv()
+		if recvErr == io.EOF {
+			t.Fatalf("ReadAllNeedles expected stream error for missing volume, got EOF")
+		}
+		if recvErr != nil {
+			if !strings.Contains(recvErr.Error(), "not found volume id") {
+				t.Fatalf("ReadAllNeedles missing-volume error mismatch: %v", recvErr)
+			}
+			break
+		}
+		if msg.GetNeedleId() == needleID && string(msg.GetNeedleBlob()) == payload {
+			seenUploadedNeedle = true
+		}
+	}
+
+	if !seenUploadedNeedle {
+		t.Fatalf("ReadAllNeedles should stream entries from existing volume before missing-volume abort")
+	}
+}
+
+func copyFileBytes(t testing.TB, grpcClient volume_server_pb.VolumeServerClient, req *volume_server_pb.CopyFileRequest) []byte {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.CopyFile(ctx, req)
+	if err != nil {
+		t.Fatalf("CopyFile start failed: %v", err)
+	}
+
+	var out []byte
+	for {
+		msg, recvErr := stream.Recv()
+		if recvErr == io.EOF {
+			return out
+		}
+		if recvErr != nil {
+			t.Fatalf("CopyFile recv failed: %v", recvErr)
+		}
+		out = append(out, msg.GetFileContent()...)
+	}
+}
+
+func findNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) {
+	t.Helper()
+
+	for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize {
+		key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize])
+		if uint64(key) != needleID {
+			continue
+		}
+		if entryOffset.IsZero() || entrySize <= 0 {
+			continue
+		}
+		return entryOffset.ToActualOffset(), int32(entrySize)
+	}
+
+	t.Fatalf("needle id %d not found in idx entries", needleID)
+	return 0, 0
+}

test/volume_server/grpc/erasure_coding_test.go

@@ -0,0 +1,777 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"io"
+	"math"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func TestEcMaintenanceModeRejections(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{
+			Maintenance: true,
+			Version:     stateResp.GetState().GetVersion(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{VolumeId: 1, Collection: ""})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeEcShardsGenerate maintenance error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       1,
+		Collection:     "",
+		SourceDataNode: "127.0.0.1:1",
+		ShardIds:       []uint32{0},
+	})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeEcShardsCopy maintenance error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+		VolumeId:   1,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeEcShardsDelete maintenance error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId:   1,
+		Collection: "",
+		FileKey:    1,
+		Version:    3,
+	})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeEcBlobDelete maintenance error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId:   1,
+		Collection: "",
+	})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeEcShardsToVolume maintenance error mismatch: %v", err)
+	}
+}
+
+func TestEcMissingInvalidAndNoopPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   98791,
+		Collection: "",
+	})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeEcShardsGenerate missing-volume error mismatch: %v", err)
+	}
+
+	rebuildResp, err := grpcClient.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{
+		VolumeId:   98792,
+		Collection: "ec-rebuild",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsRebuild missing-volume should return empty success, got: %v", err)
+	}
+	if len(rebuildResp.GetRebuiltShardIds()) != 0 {
+		t.Fatalf("VolumeEcShardsRebuild expected no rebuilt shards for missing volume, got %v", rebuildResp.GetRebuiltShardIds())
+	}
+
+	_, err = grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       98793,
+		Collection:     "ec-copy",
+		SourceDataNode: "127.0.0.1:1",
+		ShardIds:       []uint32{0},
+		DiskId:         99,
+	})
+	if err == nil || !strings.Contains(err.Error(), "invalid disk_id") {
+		t.Fatalf("VolumeEcShardsCopy invalid-disk error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+		VolumeId:   98794,
+		Collection: "ec-delete",
+		ShardIds:   []uint32{0, 1},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsDelete missing-volume should be no-op success, got: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   98795,
+		Collection: "ec-mount",
+		ShardIds:   []uint32{0},
+	})
+	if err == nil {
+		t.Fatalf("VolumeEcShardsMount should fail for missing EC shards")
+	}
+
+	_, err = grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
+		VolumeId: 98796,
+		ShardIds: []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsUnmount missing shards should be no-op success, got: %v", err)
+	}
+
+	readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
+		VolumeId: 98797,
+		ShardId:  0,
+		Offset:   0,
+		Size:     1,
+	})
+	if err == nil {
+		_, err = readStream.Recv()
+	}
+	if err == nil || err == io.EOF {
+		t.Fatalf("VolumeEcShardRead should fail for missing EC volume")
+	}
+
+	_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId:   98798,
+		Collection: "ec-blob",
+		FileKey:    1,
+		Version:    3,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete missing local EC volume should be no-op success, got: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId:   98799,
+		Collection: "ec-to-volume",
+	})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeEcShardsToVolume missing-volume error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+		VolumeId: 98800,
+	})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeEcShardsInfo missing-volume error mismatch: %v", err)
+	}
+}
+
+func TestEcGenerateMountInfoUnmountLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(115)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 990001, 0x1234ABCD)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-generate-lifecycle-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate success path failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount success path failed: %v", err)
+	}
+
+	infoResp, err := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsInfo after mount failed: %v", err)
+	}
+	if len(infoResp.GetEcShardInfos()) == 0 {
+		t.Fatalf("VolumeEcShardsInfo expected non-empty shard infos after mount")
+	}
+	if infoResp.GetVolumeSize() == 0 {
+		t.Fatalf("VolumeEcShardsInfo expected non-zero volume size after mount")
+	}
+
+	_, err = grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
+		VolumeId: volumeID,
+		ShardIds: []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsUnmount success path failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+		VolumeId: volumeID,
+	})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeEcShardsInfo after unmount expected not-found error, got: %v", err)
+	}
+}
+
+func TestEcShardReadAndBlobDeleteLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(116)
+	const fileKey = uint64(990002)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, fileKey, 0x2233CCDD)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-shard-read-delete-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount failed: %v", err)
+	}
+
+	readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
+		VolumeId: volumeID,
+		ShardId:  0,
+		Offset:   0,
+		Size:     1,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardRead start failed: %v", err)
+	}
+	firstChunk, err := readStream.Recv()
+	if err != nil {
+		t.Fatalf("VolumeEcShardRead recv failed: %v", err)
+	}
+	if len(firstChunk.GetData()) == 0 {
+		t.Fatalf("VolumeEcShardRead expected non-empty data chunk before deletion")
+	}
+
+	_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		FileKey:    fileKey,
+		Version:    uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete first delete failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		FileKey:    fileKey,
+		Version:    uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete second delete should be idempotent success, got: %v", err)
+	}
+
+	deletedStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
+		VolumeId: volumeID,
+		ShardId:  0,
+		FileKey:  fileKey,
+		Offset:   0,
+		Size:     1,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardRead deleted-check start failed: %v", err)
+	}
+	deletedMsg, err := deletedStream.Recv()
+	if err != nil {
+		t.Fatalf("VolumeEcShardRead deleted-check recv failed: %v", err)
+	}
+	if !deletedMsg.GetIsDeleted() {
+		t.Fatalf("VolumeEcShardRead expected IsDeleted=true after blob delete")
+	}
+	_, err = deletedStream.Recv()
+	if err != io.EOF {
+		t.Fatalf("VolumeEcShardRead deleted-check expected EOF after deleted marker, got: %v", err)
+	}
+}
+
+func TestEcRebuildMissingShardLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(117)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 990003, 0x3344DDEE)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-rebuild-shard-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsDelete shard 0 failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	})
+	if err == nil {
+		t.Fatalf("VolumeEcShardsMount should fail when shard 0 has been deleted")
+	}
+
+	rebuildResp, err := grpcClient.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsRebuild failed: %v", err)
+	}
+	if len(rebuildResp.GetRebuiltShardIds()) == 0 {
+		t.Fatalf("VolumeEcShardsRebuild expected rebuilt shard ids")
+	}
+	foundShard0 := false
+	for _, shardID := range rebuildResp.GetRebuiltShardIds() {
+		if shardID == 0 {
+			foundShard0 = true
+			break
+		}
+	}
+	if !foundShard0 {
+		t.Fatalf("VolumeEcShardsRebuild expected shard 0 to be rebuilt, got %v", rebuildResp.GetRebuiltShardIds())
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount shard 0 after rebuild failed: %v", err)
+	}
+}
+
+func TestEcShardsToVolumeMissingShardAndNoLiveEntries(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	httpClient := framework.NewHTTPClient()
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	t.Run("missing shard returns error", func(t *testing.T) {
+		const volumeID = uint32(118)
+		framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+		fid := framework.NewFileID(volumeID, 990004, 0x4455EEFF)
+		uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-to-volume-missing-shard-content"))
+		_ = framework.ReadAllAndClose(t, uploadResp)
+		if uploadResp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+		}
+
+		_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+		})
+		if err != nil {
+			t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+		}
+
+		_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+			ShardIds:   []uint32{0},
+		})
+		if err != nil {
+			t.Fatalf("VolumeEcShardsDelete shard 0 failed: %v", err)
+		}
+
+		_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+			ShardIds:   []uint32{1},
+		})
+		if err != nil {
+			t.Fatalf("VolumeEcShardsMount shard 1 failed: %v", err)
+		}
+
+		_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+		})
+		if err == nil || !strings.Contains(err.Error(), "missing shard 0") {
+			t.Fatalf("VolumeEcShardsToVolume missing-shard error mismatch: %v", err)
+		}
+	})
+
+	t.Run("no live entries returns failed precondition", func(t *testing.T) {
+		const volumeID = uint32(119)
+		const needleID = uint64(990005)
+		const cookie = uint32(0x5566FF11)
+		framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+		fid := framework.NewFileID(volumeID, needleID, cookie)
+		uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-no-live-entries-content"))
+		_ = framework.ReadAllAndClose(t, uploadResp)
+		if uploadResp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+		}
+
+		deleteResp := framework.DoRequest(t, httpClient, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid))
+		_ = framework.ReadAllAndClose(t, deleteResp)
+		if deleteResp.StatusCode != http.StatusAccepted {
+			t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode)
+		}
+
+		_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+		})
+		if err != nil {
+			t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+		}
+
+		_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+			ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+		})
+		if err != nil {
+			t.Fatalf("VolumeEcShardsMount data shards failed: %v", err)
+		}
+
+		_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+		})
+		if err == nil {
+			t.Fatalf("VolumeEcShardsToVolume expected failed-precondition error when no live entries")
+		}
+		if status.Code(err) != codes.FailedPrecondition {
+			t.Fatalf("VolumeEcShardsToVolume no-live-entries expected FailedPrecondition, got %v (%v)", status.Code(err), err)
+		}
+		if !strings.Contains(err.Error(), erasure_coding.EcNoLiveEntriesSubstring) {
+			t.Fatalf("VolumeEcShardsToVolume no-live-entries error should mention %q, got %v", erasure_coding.EcNoLiveEntriesSubstring, err)
+		}
+	})
+}
+
+func TestEcShardsToVolumeSuccessRoundTrip(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(120)
+	const needleID = uint64(990006)
+	const cookie = uint32(0x66771122)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	payload := []byte("ec-shards-to-volume-success-roundtrip-content")
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount data shards failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsToVolume success path failed: %v", err)
+	}
+
+	readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid)
+	readBody := framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusOK {
+		t.Fatalf("post-conversion read expected 200, got %d", readResp.StatusCode)
+	}
+	if string(readBody) != string(payload) {
+		t.Fatalf("post-conversion payload mismatch: got %q want %q", string(readBody), string(payload))
+	}
+}
+
+func TestEcShardsDeleteLastShardRemovesEcx(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(121)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 990007, 0x77882233)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-delete-all-shards-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	// Verify .ecx is present before deleting all shards.
+	ecxBeforeDelete, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Collection:         "",
+		IsEcVolume:         true,
+		Ext:                ".ecx",
+		CompactionRevision: math.MaxUint32,
+		StopOffset:         1,
+	})
+	if err != nil {
+		t.Fatalf("CopyFile .ecx before shard deletion start failed: %v", err)
+	}
+	if _, err = ecxBeforeDelete.Recv(); err != nil {
+		t.Fatalf("CopyFile .ecx before shard deletion recv failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsDelete all shards failed: %v", err)
+	}
+
+	ecxAfterDelete, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Collection:         "",
+		IsEcVolume:         true,
+		Ext:                ".ecx",
+		CompactionRevision: math.MaxUint32,
+		StopOffset:         1,
+	})
+	if err == nil {
+		_, err = ecxAfterDelete.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found ec volume id") {
+		t.Fatalf("CopyFile .ecx after deleting all shards should fail not-found, got: %v", err)
+	}
+}
+
+func TestEcShardsCopyFromPeerSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
+	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer sourceConn.Close()
+	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
+	defer destConn.Close()
+
+	const volumeID = uint32(122)
+	framework.AllocateVolume(t, sourceClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 990008, 0x88993344)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, []byte("ec-copy-from-peer-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("source upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := sourceClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("source VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	sourceDataNode := clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1]
+	_, err = destClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: sourceDataNode,
+		ShardIds:       []uint32{0},
+		CopyEcxFile:    true,
+		CopyVifFile:    true,
+	})
+	if err != nil {
+		t.Fatalf("destination VolumeEcShardsCopy success path failed: %v", err)
+	}
+
+	for _, ext := range []string{".ec00", ".ecx", ".vif"} {
+		copyStream, copyErr := destClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+			VolumeId:           volumeID,
+			Collection:         "",
+			IsEcVolume:         true,
+			Ext:                ext,
+			CompactionRevision: math.MaxUint32,
+			StopOffset:         1,
+		})
+		if copyErr != nil {
+			t.Fatalf("destination CopyFile %s start failed: %v", ext, copyErr)
+		}
+		if _, copyErr = copyStream.Recv(); copyErr != nil {
+			t.Fatalf("destination CopyFile %s recv failed: %v", ext, copyErr)
+		}
+	}
+}
+
+func TestEcShardsCopyFailsWhenSourceUnavailable(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       12345,
+		Collection:     "",
+		SourceDataNode: "127.0.0.1:1.1",
+		ShardIds:       []uint32{0},
+		CopyEcxFile:    true,
+	})
+	if err == nil || !strings.Contains(err.Error(), "VolumeEcShardsCopy volume") {
+		t.Fatalf("VolumeEcShardsCopy source-unavailable error mismatch: %v", err)
+	}
+}

test/volume_server/grpc/health_state_test.go

@@ -0,0 +1,139 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestStateAndStatusRPCs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	initialState, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	if initialState.GetState() == nil {
+		t.Fatalf("GetState returned nil state")
+	}
+
+	setResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{
+			Maintenance: true,
+			Version:     initialState.GetState().GetVersion(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("SetState(maintenance=true) failed: %v", err)
+	}
+	if !setResp.GetState().GetMaintenance() {
+		t.Fatalf("expected maintenance=true after SetState")
+	}
+
+	setResp, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{
+			Maintenance: false,
+			Version:     setResp.GetState().GetVersion(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("SetState(maintenance=false) failed: %v", err)
+	}
+	if setResp.GetState().GetMaintenance() {
+		t.Fatalf("expected maintenance=false after SetState")
+	}
+
+	statusResp, err := client.VolumeServerStatus(ctx, &volume_server_pb.VolumeServerStatusRequest{})
+	if err != nil {
+		t.Fatalf("VolumeServerStatus failed: %v", err)
+	}
+	if statusResp.GetVersion() == "" {
+		t.Fatalf("VolumeServerStatus returned empty version")
+	}
+	if len(statusResp.GetDiskStatuses()) == 0 {
+		t.Fatalf("VolumeServerStatus returned no disk statuses")
+	}
+	if statusResp.GetState() == nil {
+		t.Fatalf("VolumeServerStatus returned nil state")
+	}
+	if statusResp.GetMemoryStatus() == nil {
+		t.Fatalf("VolumeServerStatus returned nil memory status")
+	}
+	if statusResp.GetMemoryStatus().GetGoroutines() <= 0 {
+		t.Fatalf("VolumeServerStatus memory status should report goroutines, got %d", statusResp.GetMemoryStatus().GetGoroutines())
+	}
+
+	pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{})
+	if err != nil {
+		t.Fatalf("Ping failed: %v", err)
+	}
+	if pingResp.GetStartTimeNs() == 0 || pingResp.GetStopTimeNs() == 0 {
+		t.Fatalf("Ping timestamps should be non-zero: %+v", pingResp)
+	}
+	if pingResp.GetStopTimeNs() < pingResp.GetStartTimeNs() {
+		t.Fatalf("Ping stop time should be >= start time: %+v", pingResp)
+	}
+}
+
+func TestSetStateVersionMismatchAndNilStateNoop(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	initialState, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	initialVersion := initialState.GetState().GetVersion()
+
+	staleResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{
+			Maintenance: true,
+			Version:     initialVersion + 1,
+		},
+	})
+	if err == nil {
+		t.Fatalf("SetState with stale version should fail")
+	}
+	if !strings.Contains(err.Error(), "version mismatch") {
+		t.Fatalf("SetState stale version error mismatch: %v", err)
+	}
+	if staleResp.GetState().GetVersion() != initialVersion {
+		t.Fatalf("SetState stale version should not mutate server version: got %d want %d", staleResp.GetState().GetVersion(), initialVersion)
+	}
+	if staleResp.GetState().GetMaintenance() != initialState.GetState().GetMaintenance() {
+		t.Fatalf("SetState stale version should not mutate maintenance flag")
+	}
+
+	nilResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{})
+	if err != nil {
+		t.Fatalf("SetState nil-state request should be no-op success: %v", err)
+	}
+	if nilResp.GetState().GetVersion() != initialVersion {
+		t.Fatalf("SetState nil-state should keep version unchanged: got %d want %d", nilResp.GetState().GetVersion(), initialVersion)
+	}
+	if nilResp.GetState().GetMaintenance() != initialState.GetState().GetMaintenance() {
+		t.Fatalf("SetState nil-state should keep maintenance unchanged")
+	}
+}

test/volume_server/grpc/scrub_query_test.go

@@ -0,0 +1,387 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestScrubVolumeIndexAndUnsupportedMode(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(61)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	indexResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume index mode failed: %v", err)
+	}
+	if indexResp.GetTotalVolumes() != 1 {
+		t.Fatalf("ScrubVolume expected total_volumes=1, got %d", indexResp.GetTotalVolumes())
+	}
+
+	_, err = grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode(99),
+	})
+	if err == nil {
+		t.Fatalf("ScrubVolume should fail for unsupported mode")
+	}
+	if !strings.Contains(err.Error(), "unsupported volume scrub mode") {
+		t.Fatalf("ScrubVolume unsupported mode error mismatch: %v", err)
+	}
+}
+
+func TestScrubEcVolumeMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{98765},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err == nil {
+		t.Fatalf("ScrubEcVolume should fail for missing EC volume")
+	}
+	if !strings.Contains(err.Error(), "EC volume id") {
+		t.Fatalf("ScrubEcVolume missing-volume error mismatch: %v", err)
+	}
+}
+
+func TestScrubEcVolumeAutoSelectNoEcVolumes(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		Mode: volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume auto-select failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 0 {
+		t.Fatalf("ScrubEcVolume auto-select expected total_volumes=0 without EC data, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("ScrubEcVolume auto-select expected no broken volumes, got %v", resp.GetBrokenVolumeIds())
+	}
+}
+
+func TestQueryInvalidAndMissingFileIDPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	invalidStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
+		FromFileIds: []string{"bad-fid"},
+		Selections:  []string{"name"},
+		Filter:      &volume_server_pb.QueryRequest_Filter{},
+		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
+			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{},
+		},
+	})
+	if err == nil {
+		_, err = invalidStream.Recv()
+	}
+	if err == nil {
+		t.Fatalf("Query should fail for invalid file id")
+	}
+
+	missingFid := framework.NewFileID(98766, 1, 1)
+	missingStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
+		FromFileIds: []string{missingFid},
+		Selections:  []string{"name"},
+		Filter:      &volume_server_pb.QueryRequest_Filter{},
+		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
+			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{},
+		},
+	})
+	if err == nil {
+		_, err = missingStream.Recv()
+	}
+	if err == nil {
+		t.Fatalf("Query should fail for missing file id volume")
+	}
+}
+
+func TestScrubVolumeAutoSelectAndAllModes(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeIDA = uint32(62)
+	const volumeIDB = uint32(63)
+	framework.AllocateVolume(t, grpcClient, volumeIDA, "")
+	framework.AllocateVolume(t, grpcClient, volumeIDB, "")
+
+	// upload some data so index files are not zero-sized
+	httpClient := framework.NewHTTPClient()
+	framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeIDA, 1, 1), []byte("test data A"))
+	framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeIDB, 2, 2), []byte("test data B"))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	autoResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		Mode: volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume auto-select failed: %v", err)
+	}
+	if autoResp.GetTotalVolumes() < 2 {
+		t.Fatalf("ScrubVolume auto-select expected at least 2 volumes, got %d", autoResp.GetTotalVolumes())
+	}
+
+	localResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeIDA},
+		Mode:      volume_server_pb.VolumeScrubMode_LOCAL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume local mode failed: %v", err)
+	}
+	if localResp.GetTotalVolumes() != 1 {
+		t.Fatalf("ScrubVolume local mode expected total_volumes=1, got %d", localResp.GetTotalVolumes())
+	}
+	if len(localResp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("ScrubVolume local mode expected no broken volumes, got %v: %v", localResp.GetBrokenVolumeIds(), localResp.GetDetails())
+	}
+
+	fullResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeIDA},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume full mode failed: %v", err)
+	}
+	if fullResp.GetTotalVolumes() != 1 {
+		t.Fatalf("ScrubVolume full mode expected total_volumes=1, got %d", fullResp.GetTotalVolumes())
+	}
+	if len(fullResp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("ScrubVolume full mode expected no broken volumes, got %v: %v", fullResp.GetBrokenVolumeIds(), fullResp.GetDetails())
+	}
+}
+
+func TestQueryJsonSuccessAndCsvNoOutput(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(64)
+	const needleID = uint64(777001)
+	const cookie = uint32(0xAABBCCDD)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	jsonLines := []byte("{\"score\":3}\n{\"score\":12}\n{\"score\":18}\n")
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	queryStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
+		FromFileIds: []string{fid},
+		Selections:  []string{"score"},
+		Filter: &volume_server_pb.QueryRequest_Filter{
+			Field:   "score",
+			Operand: ">",
+			Value:   "10",
+		},
+		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
+			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Query json start failed: %v", err)
+	}
+
+	firstStripe, err := queryStream.Recv()
+	if err != nil {
+		t.Fatalf("Query json recv failed: %v", err)
+	}
+	records := string(firstStripe.GetRecords())
+	if !strings.Contains(records, "score:12") || !strings.Contains(records, "score:18") {
+		t.Fatalf("Query json records missing expected filtered scores: %q", records)
+	}
+	if strings.Contains(records, "score:3") {
+		t.Fatalf("Query json records should not include filtered-out score: %q", records)
+	}
+	_, err = queryStream.Recv()
+	if err != io.EOF {
+		t.Fatalf("Query json expected EOF after first stripe, got: %v", err)
+	}
+
+	csvStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
+		FromFileIds: []string{fid},
+		Selections:  []string{"score"},
+		Filter:      &volume_server_pb.QueryRequest_Filter{},
+		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
+			CsvInput: &volume_server_pb.QueryRequest_InputSerialization_CSVInput{},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Query csv start failed: %v", err)
+	}
+	_, err = csvStream.Recv()
+	if err != io.EOF {
+		t.Fatalf("Query csv expected EOF with no rows, got: %v", err)
+	}
+}
+
+func TestQueryJsonNoMatchReturnsEmptyStripe(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(65)
+	const needleID = uint64(777002)
+	const cookie = uint32(0xABABCDCD)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	jsonLines := []byte("{\"score\":1}\n{\"score\":2}\n")
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	queryStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
+		FromFileIds: []string{fid},
+		Selections:  []string{"score"},
+		Filter: &volume_server_pb.QueryRequest_Filter{
+			Field:   "score",
+			Operand: ">",
+			Value:   "100",
+		},
+		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
+			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Query json no-match start failed: %v", err)
+	}
+
+	firstStripe, err := queryStream.Recv()
+	if err != nil {
+		t.Fatalf("Query json no-match recv failed: %v", err)
+	}
+	if len(firstStripe.GetRecords()) != 0 {
+		t.Fatalf("Query json no-match expected empty records stripe, got: %q", string(firstStripe.GetRecords()))
+	}
+
+	_, err = queryStream.Recv()
+	if err != io.EOF {
+		t.Fatalf("Query json no-match expected EOF after first empty stripe, got: %v", err)
+	}
+}
+
+func TestQueryCookieMismatchReturnsEOFNoResults(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(66)
+	const needleID = uint64(777003)
+	const cookie = uint32(0xCDCDABAB)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	jsonLines := []byte("{\"score\":7}\n{\"score\":8}\n")
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	wrongCookieFid := framework.NewFileID(volumeID, needleID, cookie+1)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
+		FromFileIds: []string{wrongCookieFid},
+		Selections:  []string{"score"},
+		Filter: &volume_server_pb.QueryRequest_Filter{
+			Field:   "score",
+			Operand: ">",
+			Value:   "0",
+		},
+		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
+			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Query start for cookie mismatch should not fail immediately, got: %v", err)
+	}
+
+	_, err = stream.Recv()
+	if err != io.EOF {
+		t.Fatalf("Query cookie mismatch expected EOF with no streamed records, got: %v", err)
+	}
+}

test/volume_server/grpc/tail_test.go

@@ -0,0 +1,206 @@
+package volume_server_grpc_test
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestVolumeTailSenderMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{VolumeId: 77777, SinceNs: 0, IdleTimeoutSeconds: 1})
+	if err == nil {
+		_, err = stream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("VolumeTailSender missing-volume error mismatch: %v", err)
+	}
+}
+
+func TestVolumeTailSenderHeartbeatThenEOF(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(71)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{
+		VolumeId:           volumeID,
+		SinceNs:            0,
+		IdleTimeoutSeconds: 1,
+	})
+	if err != nil {
+		t.Fatalf("VolumeTailSender start failed: %v", err)
+	}
+
+	msg, err := stream.Recv()
+	if err != nil {
+		t.Fatalf("VolumeTailSender first recv failed: %v", err)
+	}
+	if !msg.GetIsLastChunk() {
+		t.Fatalf("expected first tail message to be heartbeat IsLastChunk=true")
+	}
+
+	_, err = stream.Recv()
+	if err != io.EOF {
+		t.Fatalf("expected EOF after idle timeout drain, got: %v", err)
+	}
+}
+
+func TestVolumeTailReceiverMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{VolumeId: 88888, SourceVolumeServer: clusterHarness.VolumeServerAddress(), SinceNs: 0, IdleTimeoutSeconds: 1})
+	if err == nil || !strings.Contains(err.Error(), "receiver not found volume") {
+		t.Fatalf("VolumeTailReceiver missing-volume error mismatch: %v", err)
+	}
+}
+
+func TestVolumeTailReceiverReplicatesSourceUpdates(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
+	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer sourceConn.Close()
+	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
+	defer destConn.Close()
+
+	const volumeID = uint32(72)
+	framework.AllocateVolume(t, sourceClient, volumeID, "")
+	framework.AllocateVolume(t, destClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 880003, 0x3456789A)
+	payload := []byte("tail-receiver-replicates-source-updates")
+
+	sourceUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, payload)
+	_ = framework.ReadAllAndClose(t, sourceUploadResp)
+	if sourceUploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("source upload expected 201, got %d", sourceUploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := destClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
+		VolumeId:           volumeID,
+		SourceVolumeServer: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1],
+		SinceNs:            0,
+		IdleTimeoutSeconds: 1,
+	})
+	if err != nil {
+		t.Fatalf("VolumeTailReceiver success path failed: %v", err)
+	}
+
+	destReadResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid)
+	destReadBody := framework.ReadAllAndClose(t, destReadResp)
+	if destReadResp.StatusCode != http.StatusOK {
+		t.Fatalf("destination read after tail receive expected 200, got %d", destReadResp.StatusCode)
+	}
+	if string(destReadBody) != string(payload) {
+		t.Fatalf("destination tail-received payload mismatch: got %q want %q", string(destReadBody), string(payload))
+	}
+}
+
+func TestVolumeTailSenderLargeNeedleChunking(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(73)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 880004, 0x456789AB)
+	largePayload := bytes.Repeat([]byte("L"), 2*1024*1024+128*1024)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, largePayload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{
+		VolumeId:           volumeID,
+		SinceNs:            0,
+		IdleTimeoutSeconds: 1,
+	})
+	if err != nil {
+		t.Fatalf("VolumeTailSender start failed: %v", err)
+	}
+
+	dataChunkCount := 0
+	sawNonLastDataChunk := false
+	sawLastDataChunk := false
+	for {
+		msg, recvErr := stream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("VolumeTailSender recv failed: %v", recvErr)
+		}
+		if len(msg.GetNeedleBody()) == 0 {
+			continue
+		}
+		dataChunkCount++
+		if msg.GetIsLastChunk() {
+			sawLastDataChunk = true
+		} else {
+			sawNonLastDataChunk = true
+		}
+	}
+
+	if dataChunkCount < 2 {
+		t.Fatalf("VolumeTailSender expected multiple chunks for large needle, got %d", dataChunkCount)
+	}
+	if !sawNonLastDataChunk {
+		t.Fatalf("VolumeTailSender expected at least one non-last data chunk")
+	}
+	if !sawLastDataChunk {
+		t.Fatalf("VolumeTailSender expected a final data chunk marked IsLastChunk=true")
+	}
+}

test/volume_server/grpc/tiering_remote_test.go

@@ -0,0 +1,236 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestFetchAndWriteNeedleMaintenanceAndMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{
+		VolumeId: 98781,
+		NeedleId: 1,
+	})
+	if err == nil || !strings.Contains(err.Error(), "not found volume id") {
+		t.Fatalf("FetchAndWriteNeedle missing-volume error mismatch: %v", err)
+	}
+
+	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{
+			Maintenance: true,
+			Version:     stateResp.GetState().GetVersion(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	_, err = grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{
+		VolumeId: 1,
+		NeedleId: 1,
+	})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("FetchAndWriteNeedle maintenance error mismatch: %v", err)
+	}
+}
+
+func TestFetchAndWriteNeedleInvalidRemoteConfig(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(88)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{
+		VolumeId: volumeID,
+		NeedleId: 1,
+		Cookie:   1,
+		Size:     1,
+		RemoteConf: &remote_pb.RemoteConf{
+			Name: "it-invalid-remote",
+			Type: "does-not-exist",
+		},
+		RemoteLocation: &remote_pb.RemoteStorageLocation{
+			Name: "it-invalid-remote",
+			Path: "/test",
+		},
+	})
+	if err == nil || !strings.Contains(err.Error(), "get remote client") {
+		t.Fatalf("FetchAndWriteNeedle invalid-remote error mismatch: %v", err)
+	}
+}
+
+func TestVolumeTierMoveDatToRemoteErrorPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(85)
+	const collection = "tier-collection"
+	framework.AllocateVolume(t, grpcClient, volumeID, collection)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	missingStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{
+		VolumeId:               98782,
+		Collection:             collection,
+		DestinationBackendName: "dummy",
+	})
+	if err == nil {
+		_, err = missingStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeTierMoveDatToRemote missing-volume error mismatch: %v", err)
+	}
+
+	mismatchStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{
+		VolumeId:               volumeID,
+		Collection:             "wrong-collection",
+		DestinationBackendName: "dummy",
+	})
+	if err == nil {
+		_, err = mismatchStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "unexpected input") {
+		t.Fatalf("VolumeTierMoveDatToRemote collection mismatch error mismatch: %v", err)
+	}
+
+	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{
+			Maintenance: true,
+			Version:     stateResp.GetState().GetVersion(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	maintenanceStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{
+		VolumeId:               volumeID,
+		Collection:             collection,
+		DestinationBackendName: "dummy",
+	})
+	if err == nil {
+		_, err = maintenanceStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeTierMoveDatToRemote maintenance error mismatch: %v", err)
+	}
+}
+
+func TestVolumeTierMoveDatToRemoteMissingBackend(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(89)
+	const collection = "tier-missing-backend"
+	framework.AllocateVolume(t, grpcClient, volumeID, collection)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{
+		VolumeId:               volumeID,
+		Collection:             collection,
+		DestinationBackendName: "definitely-missing-backend",
+	})
+	if err == nil {
+		_, err = stream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "destination definitely-missing-backend not found") {
+		t.Fatalf("VolumeTierMoveDatToRemote missing-backend error mismatch: %v", err)
+	}
+}
+
+func TestVolumeTierMoveDatFromRemoteErrorPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(86)
+	const collection = "tier-download-collection"
+	framework.AllocateVolume(t, grpcClient, volumeID, collection)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	missingStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{
+		VolumeId:   98783,
+		Collection: collection,
+	})
+	if err == nil {
+		_, err = missingStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeTierMoveDatFromRemote missing-volume error mismatch: %v", err)
+	}
+
+	mismatchStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{
+		VolumeId:   volumeID,
+		Collection: "wrong-collection",
+	})
+	if err == nil {
+		_, err = mismatchStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "unexpected input") {
+		t.Fatalf("VolumeTierMoveDatFromRemote collection mismatch error mismatch: %v", err)
+	}
+
+	localDiskStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{
+		VolumeId:   volumeID,
+		Collection: collection,
+	})
+	if err == nil {
+		_, err = localDiskStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "already on local disk") {
+		t.Fatalf("VolumeTierMoveDatFromRemote local-disk error mismatch: %v", err)
+	}
+}

test/volume_server/grpc/vacuum_test.go

@@ -0,0 +1,87 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestVacuumVolumeCheckSuccessAndMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(31)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VacuumVolumeCheck existing volume failed: %v", err)
+	}
+	if resp.GetGarbageRatio() < 0 || resp.GetGarbageRatio() > 1 {
+		t.Fatalf("unexpected garbage ratio: %f", resp.GetGarbageRatio())
+	}
+
+	_, err = grpcClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{VolumeId: 99999})
+	if err == nil {
+		t.Fatalf("VacuumVolumeCheck should fail for missing volume")
+	}
+}
+
+func TestVacuumMaintenanceModeRejections(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	assertMaintenanceErr := func(name string, err error) {
+		t.Helper()
+		if err == nil {
+			t.Fatalf("%s should fail in maintenance mode", name)
+		}
+		if !strings.Contains(err.Error(), "maintenance mode") {
+			t.Fatalf("%s expected maintenance mode error, got: %v", name, err)
+		}
+	}
+
+	compactStream, err := grpcClient.VacuumVolumeCompact(ctx, &volume_server_pb.VacuumVolumeCompactRequest{VolumeId: 31})
+	if err == nil {
+		_, err = compactStream.Recv()
+	}
+	assertMaintenanceErr("VacuumVolumeCompact", err)
+
+	_, err = grpcClient.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: 31})
+	assertMaintenanceErr("VacuumVolumeCommit", err)
+
+	_, err = grpcClient.VacuumVolumeCleanup(ctx, &volume_server_pb.VacuumVolumeCleanupRequest{VolumeId: 31})
+	assertMaintenanceErr("VacuumVolumeCleanup", err)
+}

test/volume_server/http/admin_test.go

@@ -0,0 +1,174 @@
+package volume_server_http_test
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
+)
+
+func TestAdminStatusAndHealthz(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	client := framework.NewHTTPClient()
+
+	statusReq, err := http.NewRequest(http.MethodGet, cluster.VolumeAdminURL()+"/status", nil)
+	if err != nil {
+		t.Fatalf("create status request: %v", err)
+	}
+	statusReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-1")
+
+	statusResp := framework.DoRequest(t, client, statusReq)
+	statusBody := framework.ReadAllAndClose(t, statusResp)
+
+	if statusResp.StatusCode != http.StatusOK {
+		t.Fatalf("expected /status code 200, got %d, body: %s", statusResp.StatusCode, string(statusBody))
+	}
+	if got := statusResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
+		t.Fatalf("expected /status Server header to contain SeaweedFS Volume, got %q", got)
+	}
+	if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-1" {
+		t.Fatalf("expected echoed request id, got %q", got)
+	}
+
+	var payload map[string]interface{}
+	if err := json.Unmarshal(statusBody, &payload); err != nil {
+		t.Fatalf("decode status response: %v", err)
+	}
+	for _, field := range []string{"Version", "DiskStatuses", "Volumes"} {
+		if _, found := payload[field]; !found {
+			t.Fatalf("status payload missing field %q", field)
+		}
+	}
+
+	healthReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/healthz")
+	healthReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-2")
+	healthResp := framework.DoRequest(t, client, healthReq)
+	_ = framework.ReadAllAndClose(t, healthResp)
+	if healthResp.StatusCode != http.StatusOK {
+		t.Fatalf("expected /healthz code 200, got %d", healthResp.StatusCode)
+	}
+	if got := healthResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
+		t.Fatalf("expected /healthz Server header to contain SeaweedFS Volume, got %q", got)
+	}
+	if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-2" {
+		t.Fatalf("expected /healthz echoed request id, got %q", got)
+	}
+
+	uiResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
+	uiBody := framework.ReadAllAndClose(t, uiResp)
+	if uiResp.StatusCode != http.StatusOK {
+		t.Fatalf("expected /ui/index.html code 200, got %d, body: %s", uiResp.StatusCode, string(uiBody))
+	}
+	if !strings.Contains(strings.ToLower(string(uiBody)), "volume") {
+		t.Fatalf("ui page does not look like volume status page")
+	}
+}
+
+func TestOptionsMethodsByPort(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P2())
+	client := framework.NewHTTPClient()
+
+	adminResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/"))
+	_ = framework.ReadAllAndClose(t, adminResp)
+	if adminResp.StatusCode != http.StatusOK {
+		t.Fatalf("admin OPTIONS expected 200, got %d", adminResp.StatusCode)
+	}
+	adminAllowed := adminResp.Header.Get("Access-Control-Allow-Methods")
+	for _, expected := range []string{"PUT", "POST", "GET", "DELETE", "OPTIONS"} {
+		if !strings.Contains(adminAllowed, expected) {
+			t.Fatalf("admin allow methods missing %q, got %q", expected, adminAllowed)
+		}
+	}
+	if adminResp.Header.Get("Access-Control-Allow-Headers") != "*" {
+		t.Fatalf("admin allow headers expected '*', got %q", adminResp.Header.Get("Access-Control-Allow-Headers"))
+	}
+
+	publicResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodOptions, cluster.VolumePublicURL()+"/"))
+	_ = framework.ReadAllAndClose(t, publicResp)
+	if publicResp.StatusCode != http.StatusOK {
+		t.Fatalf("public OPTIONS expected 200, got %d", publicResp.StatusCode)
+	}
+	publicAllowed := publicResp.Header.Get("Access-Control-Allow-Methods")
+	if !strings.Contains(publicAllowed, "GET") || !strings.Contains(publicAllowed, "OPTIONS") {
+		t.Fatalf("public allow methods expected GET and OPTIONS, got %q", publicAllowed)
+	}
+	if strings.Contains(publicAllowed, "POST") {
+		t.Fatalf("public allow methods should not include POST, got %q", publicAllowed)
+	}
+	if publicResp.Header.Get("Access-Control-Allow-Headers") != "*" {
+		t.Fatalf("public allow headers expected '*', got %q", publicResp.Header.Get("Access-Control-Allow-Headers"))
+	}
+}
+
+func TestOptionsWithOriginIncludesCorsHeaders(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P2())
+	client := framework.NewHTTPClient()
+
+	adminReq := mustNewRequest(t, http.MethodOptions, cluster.VolumeAdminURL()+"/")
+	adminReq.Header.Set("Origin", "https://example.com")
+	adminResp := framework.DoRequest(t, client, adminReq)
+	_ = framework.ReadAllAndClose(t, adminResp)
+	if adminResp.StatusCode != http.StatusOK {
+		t.Fatalf("admin OPTIONS expected 200, got %d", adminResp.StatusCode)
+	}
+	if adminResp.Header.Get("Access-Control-Allow-Origin") != "*" {
+		t.Fatalf("admin OPTIONS expected Access-Control-Allow-Origin=*, got %q", adminResp.Header.Get("Access-Control-Allow-Origin"))
+	}
+	if adminResp.Header.Get("Access-Control-Allow-Credentials") != "true" {
+		t.Fatalf("admin OPTIONS expected Access-Control-Allow-Credentials=true, got %q", adminResp.Header.Get("Access-Control-Allow-Credentials"))
+	}
+
+	publicReq := mustNewRequest(t, http.MethodOptions, cluster.VolumePublicURL()+"/")
+	publicReq.Header.Set("Origin", "https://example.com")
+	publicResp := framework.DoRequest(t, client, publicReq)
+	_ = framework.ReadAllAndClose(t, publicResp)
+	if publicResp.StatusCode != http.StatusOK {
+		t.Fatalf("public OPTIONS expected 200, got %d", publicResp.StatusCode)
+	}
+	if publicResp.Header.Get("Access-Control-Allow-Origin") != "*" {
+		t.Fatalf("public OPTIONS expected Access-Control-Allow-Origin=*, got %q", publicResp.Header.Get("Access-Control-Allow-Origin"))
+	}
+	if publicResp.Header.Get("Access-Control-Allow-Credentials") != "true" {
+		t.Fatalf("public OPTIONS expected Access-Control-Allow-Credentials=true, got %q", publicResp.Header.Get("Access-Control-Allow-Credentials"))
+	}
+}
+
+func TestUiIndexNotExposedWhenJwtSigningEnabled(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P3())
+	client := framework.NewHTTPClient()
+
+	resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
+	body := framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("expected /ui/index.html to be gated by auth under JWT profile (401), got %d body=%s", resp.StatusCode, string(body))
+	}
+}
+
+func mustNewRequest(t testing.TB, method, url string) *http.Request {
+	t.Helper()
+	req, err := http.NewRequest(method, url, nil)
+	if err != nil {
+		t.Fatalf("create request %s %s: %v", method, url, err)
+	}
+	return req
+}

test/volume_server/http/auth_test.go

@@ -0,0 +1,419 @@
+package volume_server_http_test
+
+import (
+	"bytes"
+	"net/http"
+	"testing"
+	"time"
+
+	jwt "github.com/golang-jwt/jwt/v5"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+)
+
+func TestJWTAuthForWriteAndRead(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P3()
+	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(51)
+	const needleID = uint64(123456)
+	const cookie = uint32(0xABCDEF12)
+
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	payload := []byte("jwt-protected-content")
+	client := framework.NewHTTPClient()
+
+	unauthWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
+	unauthWriteResp := framework.DoRequest(t, client, unauthWrite)
+	_ = framework.ReadAllAndClose(t, unauthWriteResp)
+	if unauthWriteResp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("unauthorized write expected 401, got %d", unauthWriteResp.StatusCode)
+	}
+
+	invalidWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
+	invalidWrite.Header.Set("Authorization", "Bearer invalid")
+	invalidWriteResp := framework.DoRequest(t, client, invalidWrite)
+	_ = framework.ReadAllAndClose(t, invalidWriteResp)
+	if invalidWriteResp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("invalid write token expected 401, got %d", invalidWriteResp.StatusCode)
+	}
+
+	writeToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
+	authWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
+	authWrite.Header.Set("Authorization", "Bearer "+string(writeToken))
+	authWriteResp := framework.DoRequest(t, client, authWrite)
+	_ = framework.ReadAllAndClose(t, authWriteResp)
+	if authWriteResp.StatusCode != http.StatusCreated {
+		t.Fatalf("authorized write expected 201, got %d", authWriteResp.StatusCode)
+	}
+
+	unauthReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	unauthReadResp := framework.DoRequest(t, client, unauthReadReq)
+	_ = framework.ReadAllAndClose(t, unauthReadResp)
+	if unauthReadResp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("unauthorized read expected 401, got %d", unauthReadResp.StatusCode)
+	}
+
+	readToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid)
+	authReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	authReadReq.Header.Set("Authorization", "Bearer "+string(readToken))
+	authReadResp := framework.DoRequest(t, client, authReadReq)
+	authReadBody := framework.ReadAllAndClose(t, authReadResp)
+	if authReadResp.StatusCode != http.StatusOK {
+		t.Fatalf("authorized read expected 200, got %d", authReadResp.StatusCode)
+	}
+	if string(authReadBody) != string(payload) {
+		t.Fatalf("authorized read content mismatch: got %q want %q", string(authReadBody), string(payload))
+	}
+}
+
+func TestJWTAuthRejectsFidMismatch(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P3()
+	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(52)
+	const needleID = uint64(223344)
+	const cookie = uint32(0x10203040)
+	const otherNeedleID = uint64(223345)
+	const otherCookie = uint32(0x50607080)
+	const wrongCookie = uint32(0x10203041)
+
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	otherFid := framework.NewFileID(volumeID, otherNeedleID, otherCookie)
+	payload := []byte("jwt-fid-mismatch-content")
+	client := framework.NewHTTPClient()
+
+	writeTokenForOtherFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFid)
+	mismatchedWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
+	mismatchedWrite.Header.Set("Authorization", "Bearer "+string(writeTokenForOtherFid))
+	mismatchedWriteResp := framework.DoRequest(t, client, mismatchedWrite)
+	_ = framework.ReadAllAndClose(t, mismatchedWriteResp)
+	if mismatchedWriteResp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("write with mismatched fid token expected 401, got %d", mismatchedWriteResp.StatusCode)
+	}
+
+	wrongCookieFid := framework.NewFileID(volumeID, needleID, wrongCookie)
+	writeTokenWrongCookie := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, wrongCookieFid)
+	wrongCookieWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
+	wrongCookieWrite.Header.Set("Authorization", "Bearer "+string(writeTokenWrongCookie))
+	wrongCookieWriteResp := framework.DoRequest(t, client, wrongCookieWrite)
+	_ = framework.ReadAllAndClose(t, wrongCookieWriteResp)
+	if wrongCookieWriteResp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("write with wrong-cookie fid token expected 401, got %d", wrongCookieWriteResp.StatusCode)
+	}
+
+	writeTokenForFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
+	validWrite := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
+	validWrite.Header.Set("Authorization", "Bearer "+string(writeTokenForFid))
+	validWriteResp := framework.DoRequest(t, client, validWrite)
+	_ = framework.ReadAllAndClose(t, validWriteResp)
+	if validWriteResp.StatusCode != http.StatusCreated {
+		t.Fatalf("authorized write expected 201, got %d", validWriteResp.StatusCode)
+	}
+
+	readTokenForOtherFid := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFid)
+	mismatchedReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	mismatchedReadReq.Header.Set("Authorization", "Bearer "+string(readTokenForOtherFid))
+	mismatchedReadResp := framework.DoRequest(t, client, mismatchedReadReq)
+	_ = framework.ReadAllAndClose(t, mismatchedReadResp)
+	if mismatchedReadResp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("read with mismatched fid token expected 401, got %d", mismatchedReadResp.StatusCode)
+	}
+
+	readTokenWrongCookie := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, wrongCookieFid)
+	wrongCookieReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	wrongCookieReadReq.Header.Set("Authorization", "Bearer "+string(readTokenWrongCookie))
+	wrongCookieReadResp := framework.DoRequest(t, client, wrongCookieReadReq)
+	_ = framework.ReadAllAndClose(t, wrongCookieReadResp)
+	if wrongCookieReadResp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("read with wrong-cookie fid token expected 401, got %d", wrongCookieReadResp.StatusCode)
+	}
+}
+
+func newUploadRequest(t testing.TB, url string, payload []byte) *http.Request {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(payload))
+	if err != nil {
+		t.Fatalf("create upload request %s: %v", url, err)
+	}
+	req.Header.Set("Content-Type", "application/octet-stream")
+	return req
+}
+
+func TestJWTAuthRejectsExpiredTokens(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P3()
+	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(53)
+	const needleID = uint64(334455)
+	const cookie = uint32(0x22334455)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	payload := []byte("expired-token-content")
+	client := framework.NewHTTPClient()
+
+	expiredWriteToken := mustGenExpiredToken(t, []byte(profile.JWTSigningKey), fid)
+	writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
+	writeReq.Header.Set("Authorization", "Bearer "+expiredWriteToken)
+	writeResp := framework.DoRequest(t, client, writeReq)
+	_ = framework.ReadAllAndClose(t, writeResp)
+	if writeResp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("expired write token expected 401, got %d", writeResp.StatusCode)
+	}
+
+	// Seed data with a valid token so read auth path can be exercised against existing content.
+	validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
+	validWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
+	validWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken))
+	validWriteResp := framework.DoRequest(t, client, validWriteReq)
+	_ = framework.ReadAllAndClose(t, validWriteResp)
+	if validWriteResp.StatusCode != http.StatusCreated {
+		t.Fatalf("valid write expected 201, got %d", validWriteResp.StatusCode)
+	}
+
+	expiredReadToken := mustGenExpiredToken(t, []byte(profile.JWTReadKey), fid)
+	readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	readReq.Header.Set("Authorization", "Bearer "+expiredReadToken)
+	readResp := framework.DoRequest(t, client, readReq)
+	_ = framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("expired read token expected 401, got %d", readResp.StatusCode)
+	}
+}
+
+func TestJWTAuthViaQueryParamAndCookie(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P3()
+	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(54)
+	const needleID = uint64(445566)
+	const cookie = uint32(0x31415926)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	payload := []byte("jwt-query-cookie-content")
+	client := framework.NewHTTPClient()
+
+	writeToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
+	writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(writeToken), payload)
+	writeResp := framework.DoRequest(t, client, writeReq)
+	_ = framework.ReadAllAndClose(t, writeResp)
+	if writeResp.StatusCode != http.StatusCreated {
+		t.Fatalf("query-jwt write expected 201, got %d", writeResp.StatusCode)
+	}
+
+	readToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid)
+	readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(readToken)})
+	readResp := framework.DoRequest(t, client, readReq)
+	readBody := framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusOK {
+		t.Fatalf("cookie-jwt read expected 200, got %d", readResp.StatusCode)
+	}
+	if string(readBody) != string(payload) {
+		t.Fatalf("cookie-jwt read body mismatch: got %q want %q", string(readBody), string(payload))
+	}
+}
+
+func TestJWTTokenSourcePrecedenceQueryOverHeader(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P3()
+	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(55)
+	const needleID = uint64(556677)
+	const cookie = uint32(0x99887766)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1)
+	payload := []byte("jwt-precedence-content")
+	client := framework.NewHTTPClient()
+
+	validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
+	invalidWriteQueryToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID)
+	writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidWriteQueryToken), payload)
+	writeReq.Header.Set("Authorization", "Bearer "+string(validWriteToken))
+	writeResp := framework.DoRequest(t, client, writeReq)
+	_ = framework.ReadAllAndClose(t, writeResp)
+	if writeResp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("query token should take precedence over header token for write, expected 401 got %d", writeResp.StatusCode)
+	}
+
+	// Seed data with valid write token, then exercise read precedence.
+	seedWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
+	seedWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken))
+	seedWriteResp := framework.DoRequest(t, client, seedWriteReq)
+	_ = framework.ReadAllAndClose(t, seedWriteResp)
+	if seedWriteResp.StatusCode != http.StatusCreated {
+		t.Fatalf("seed write expected 201, got %d", seedWriteResp.StatusCode)
+	}
+
+	validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid)
+	invalidReadQueryToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID)
+	readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidReadQueryToken))
+	readReq.Header.Set("Authorization", "Bearer "+string(validReadToken))
+	readResp := framework.DoRequest(t, client, readReq)
+	_ = framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("query token should take precedence over header token for read, expected 401 got %d", readResp.StatusCode)
+	}
+}
+
+func TestJWTTokenSourcePrecedenceHeaderOverCookie(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P3()
+	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(56)
+	const needleID = uint64(667788)
+	const cookie = uint32(0x11229988)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1)
+	payload := []byte("jwt-precedence-header-cookie")
+	client := framework.NewHTTPClient()
+
+	validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
+	invalidCookieWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID)
+	writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
+	writeReq.Header.Set("Authorization", "Bearer "+string(validWriteToken))
+	writeReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieWriteToken)})
+	writeResp := framework.DoRequest(t, client, writeReq)
+	_ = framework.ReadAllAndClose(t, writeResp)
+	if writeResp.StatusCode != http.StatusCreated {
+		t.Fatalf("header token should take precedence over cookie token for write, expected 201 got %d", writeResp.StatusCode)
+	}
+
+	validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid)
+	invalidCookieReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID)
+	readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	readReq.Header.Set("Authorization", "Bearer "+string(validReadToken))
+	readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieReadToken)})
+	readResp := framework.DoRequest(t, client, readReq)
+	readBody := framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusOK {
+		t.Fatalf("header token should take precedence over cookie token for read, expected 200 got %d", readResp.StatusCode)
+	}
+	if string(readBody) != string(payload) {
+		t.Fatalf("header-over-cookie read body mismatch: got %q want %q", string(readBody), string(payload))
+	}
+}
+
+func TestJWTTokenSourcePrecedenceQueryOverCookie(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P3()
+	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(57)
+	const needleID = uint64(778899)
+	const cookie = uint32(0x88776655)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	otherFID := framework.NewFileID(volumeID, needleID+1, cookie+1)
+	payload := []byte("jwt-precedence-query-cookie")
+	client := framework.NewHTTPClient()
+
+	validWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, fid)
+	invalidQueryWriteToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTSigningKey)), 60, otherFID)
+	writeReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidQueryWriteToken), payload)
+	writeReq.AddCookie(&http.Cookie{Name: "AT", Value: string(validWriteToken)})
+	writeResp := framework.DoRequest(t, client, writeReq)
+	_ = framework.ReadAllAndClose(t, writeResp)
+	if writeResp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("query token should take precedence over cookie token for write, expected 401 got %d", writeResp.StatusCode)
+	}
+
+	// Seed data with valid write token so read precedence can be exercised.
+	seedWriteReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
+	seedWriteReq.Header.Set("Authorization", "Bearer "+string(validWriteToken))
+	seedWriteResp := framework.DoRequest(t, client, seedWriteReq)
+	_ = framework.ReadAllAndClose(t, seedWriteResp)
+	if seedWriteResp.StatusCode != http.StatusCreated {
+		t.Fatalf("seed write expected 201, got %d", seedWriteResp.StatusCode)
+	}
+
+	validReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, fid)
+	invalidQueryReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID)
+	readReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(invalidQueryReadToken))
+	readReq.AddCookie(&http.Cookie{Name: "AT", Value: string(validReadToken)})
+	readResp := framework.DoRequest(t, client, readReq)
+	_ = framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("query token should take precedence over cookie token for read, expected 401 got %d", readResp.StatusCode)
+	}
+
+	// Validate positive path: valid query token should succeed even if cookie token is invalid.
+	validQueryReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?jwt="+string(validReadToken))
+	invalidCookieReadToken := security.GenJwtForVolumeServer(security.SigningKey([]byte(profile.JWTReadKey)), 60, otherFID)
+	validQueryReadReq.AddCookie(&http.Cookie{Name: "AT", Value: string(invalidCookieReadToken)})
+	validQueryReadResp := framework.DoRequest(t, client, validQueryReadReq)
+	validQueryReadBody := framework.ReadAllAndClose(t, validQueryReadResp)
+	if validQueryReadResp.StatusCode != http.StatusOK {
+		t.Fatalf("valid query token should succeed over invalid cookie token, expected 200 got %d", validQueryReadResp.StatusCode)
+	}
+	if string(validQueryReadBody) != string(payload) {
+		t.Fatalf("query-over-cookie read body mismatch: got %q want %q", string(validQueryReadBody), string(payload))
+	}
+}
+
+func mustGenExpiredToken(t testing.TB, key []byte, fid string) string {
+	t.Helper()
+	claims := security.SeaweedFileIdClaims{
+		Fid: fid,
+		RegisteredClaims: jwt.RegisteredClaims{
+			ExpiresAt: jwt.NewNumericDate(time.Now().Add(-1 * time.Minute)),
+		},
+	}
+	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
+	signed, err := token.SignedString(key)
+	if err != nil {
+		t.Fatalf("sign expired token: %v", err)
+	}
+	return signed
+}

test/volume_server/http/chunk_manifest_test.go

@@ -0,0 +1,232 @@
+package volume_server_http_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+)
+
+func TestChunkManifestExpansionAndBypass(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(102)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+
+	chunkFID := framework.NewFileID(volumeID, 772005, 0x5E6F7081)
+	chunkPayload := []byte("chunk-manifest-expanded-content")
+	chunkUploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID, chunkPayload)
+	_ = framework.ReadAllAndClose(t, chunkUploadResp)
+	if chunkUploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("chunk upload expected 201, got %d", chunkUploadResp.StatusCode)
+	}
+
+	manifest := &operation.ChunkManifest{
+		Name: "manifest.bin",
+		Mime: "application/octet-stream",
+		Size: int64(len(chunkPayload)),
+		Chunks: []*operation.ChunkInfo{
+			{
+				Fid:    chunkFID,
+				Offset: 0,
+				Size:   int64(len(chunkPayload)),
+			},
+		},
+	}
+	manifestBytes, err := json.Marshal(manifest)
+	if err != nil {
+		t.Fatalf("marshal chunk manifest: %v", err)
+	}
+
+	manifestFID := framework.NewFileID(volumeID, 772006, 0x6F708192)
+	manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes))
+	if err != nil {
+		t.Fatalf("create manifest upload request: %v", err)
+	}
+	manifestUploadReq.Header.Set("Content-Type", "application/json")
+	manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq)
+	_ = framework.ReadAllAndClose(t, manifestUploadResp)
+	if manifestUploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode)
+	}
+
+	expandedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), manifestFID)
+	expandedReadBody := framework.ReadAllAndClose(t, expandedReadResp)
+	if expandedReadResp.StatusCode != http.StatusOK {
+		t.Fatalf("manifest expanded read expected 200, got %d", expandedReadResp.StatusCode)
+	}
+	if string(expandedReadBody) != string(chunkPayload) {
+		t.Fatalf("manifest expanded read mismatch: got %q want %q", string(expandedReadBody), string(chunkPayload))
+	}
+	if expandedReadResp.Header.Get("X-File-Store") != "chunked" {
+		t.Fatalf("manifest expanded read expected X-File-Store=chunked, got %q", expandedReadResp.Header.Get("X-File-Store"))
+	}
+
+	bypassReadResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=false"))
+	bypassReadBody := framework.ReadAllAndClose(t, bypassReadResp)
+	if bypassReadResp.StatusCode != http.StatusOK {
+		t.Fatalf("manifest bypass read expected 200, got %d", bypassReadResp.StatusCode)
+	}
+	if bypassReadResp.Header.Get("X-File-Store") != "" {
+		t.Fatalf("manifest bypass read expected empty X-File-Store header, got %q", bypassReadResp.Header.Get("X-File-Store"))
+	}
+
+	var gotManifest operation.ChunkManifest
+	if err = json.Unmarshal(bypassReadBody, &gotManifest); err != nil {
+		t.Fatalf("manifest bypass read expected JSON payload, got decode error: %v body=%q", err, string(bypassReadBody))
+	}
+	if len(gotManifest.Chunks) != 1 || gotManifest.Chunks[0].Fid != chunkFID {
+		t.Fatalf("manifest bypass read payload mismatch: %+v", gotManifest)
+	}
+}
+
+func TestChunkManifestDeleteRemovesChildChunks(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(104)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+
+	chunkFID := framework.NewFileID(volumeID, 772008, 0x8192A3B4)
+	chunkPayload := []byte("chunk-manifest-delete-content")
+	chunkUploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID, chunkPayload)
+	_ = framework.ReadAllAndClose(t, chunkUploadResp)
+	if chunkUploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("chunk upload expected 201, got %d", chunkUploadResp.StatusCode)
+	}
+
+	manifest := &operation.ChunkManifest{
+		Name: "manifest-delete.bin",
+		Mime: "application/octet-stream",
+		Size: int64(len(chunkPayload)),
+		Chunks: []*operation.ChunkInfo{
+			{
+				Fid:    chunkFID,
+				Offset: 0,
+				Size:   int64(len(chunkPayload)),
+			},
+		},
+	}
+	manifestBytes, err := json.Marshal(manifest)
+	if err != nil {
+		t.Fatalf("marshal chunk manifest: %v", err)
+	}
+
+	manifestFID := framework.NewFileID(volumeID, 772009, 0x92A3B4C5)
+	manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes))
+	if err != nil {
+		t.Fatalf("create manifest upload request: %v", err)
+	}
+	manifestUploadReq.Header.Set("Content-Type", "application/json")
+	manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq)
+	_ = framework.ReadAllAndClose(t, manifestUploadResp)
+	if manifestUploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode)
+	}
+
+	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+manifestFID))
+	deleteBody := framework.ReadAllAndClose(t, deleteResp)
+	if deleteResp.StatusCode != http.StatusAccepted {
+		t.Fatalf("manifest delete expected 202, got %d", deleteResp.StatusCode)
+	}
+	var deleteResult map[string]int64
+	if err = json.Unmarshal(deleteBody, &deleteResult); err != nil {
+		t.Fatalf("decode manifest delete response: %v body=%q", err, string(deleteBody))
+	}
+	if deleteResult["size"] != int64(len(chunkPayload)) {
+		t.Fatalf("manifest delete expected size=%d, got %d", len(chunkPayload), deleteResult["size"])
+	}
+
+	manifestReadAfterDelete := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), manifestFID)
+	_ = framework.ReadAllAndClose(t, manifestReadAfterDelete)
+	if manifestReadAfterDelete.StatusCode != http.StatusNotFound {
+		t.Fatalf("manifest read after delete expected 404, got %d", manifestReadAfterDelete.StatusCode)
+	}
+
+	chunkReadAfterDelete := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), chunkFID)
+	_ = framework.ReadAllAndClose(t, chunkReadAfterDelete)
+	if chunkReadAfterDelete.StatusCode != http.StatusNotFound {
+		t.Fatalf("chunk read after manifest delete expected 404, got %d", chunkReadAfterDelete.StatusCode)
+	}
+}
+
+func TestChunkManifestDeleteFailsWhenChildDeletionFails(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(105)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	manifest := &operation.ChunkManifest{
+		Name: "manifest-delete-failure.bin",
+		Mime: "application/octet-stream",
+		Size: 1,
+		Chunks: []*operation.ChunkInfo{
+			{
+				Fid:    "not-a-valid-fid",
+				Offset: 0,
+				Size:   1,
+			},
+		},
+	}
+	manifestBytes, err := json.Marshal(manifest)
+	if err != nil {
+		t.Fatalf("marshal chunk manifest: %v", err)
+	}
+
+	manifestFID := framework.NewFileID(volumeID, 772010, 0xA3B4C5D6)
+	manifestUploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=true", bytes.NewReader(manifestBytes))
+	if err != nil {
+		t.Fatalf("create manifest upload request: %v", err)
+	}
+	manifestUploadReq.Header.Set("Content-Type", "application/json")
+	manifestUploadResp := framework.DoRequest(t, client, manifestUploadReq)
+	_ = framework.ReadAllAndClose(t, manifestUploadResp)
+	if manifestUploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("manifest upload expected 201, got %d", manifestUploadResp.StatusCode)
+	}
+
+	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+manifestFID))
+	deleteBody := framework.ReadAllAndClose(t, deleteResp)
+	if deleteResp.StatusCode != http.StatusInternalServerError {
+		t.Fatalf("manifest delete with invalid child fid expected 500, got %d body=%q", deleteResp.StatusCode, string(deleteBody))
+	}
+
+	manifestBypassRead := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+manifestFID+"?cm=false"))
+	manifestBypassBody := framework.ReadAllAndClose(t, manifestBypassRead)
+	if manifestBypassRead.StatusCode != http.StatusOK {
+		t.Fatalf("manifest bypass read after failed delete expected 200, got %d", manifestBypassRead.StatusCode)
+	}
+	var gotManifest operation.ChunkManifest
+	if err = json.Unmarshal(manifestBypassBody, &gotManifest); err != nil {
+		t.Fatalf("manifest bypass read expected JSON payload, got decode error: %v body=%q", err, string(manifestBypassBody))
+	}
+	if len(gotManifest.Chunks) != 1 || gotManifest.Chunks[0].Fid != "not-a-valid-fid" {
+		t.Fatalf("manifest payload mismatch after failed delete: %+v", gotManifest)
+	}
+}

test/volume_server/http/compressed_read_test.go

@@ -0,0 +1,97 @@
+package volume_server_http_test
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"net/http"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+func gzipData(t testing.TB, data []byte) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	if _, err := zw.Write(data); err != nil {
+		t.Fatalf("gzip write: %v", err)
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatalf("gzip close: %v", err)
+	}
+	return buf.Bytes()
+}
+
+func gunzipData(t testing.TB, data []byte) []byte {
+	t.Helper()
+	zr, err := gzip.NewReader(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("gunzip new reader: %v", err)
+	}
+	defer zr.Close()
+	out, err := io.ReadAll(zr)
+	if err != nil {
+		t.Fatalf("gunzip read: %v", err)
+	}
+	return out
+}
+
+func TestCompressedReadAcceptEncodingMatrix(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(103)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 772007, 0x708192A3)
+	plainPayload := []byte("compressed-read-accept-encoding-matrix-content-compressed-read-accept-encoding-matrix-content")
+	compressedPayload := gzipData(t, plainPayload)
+
+	uploadReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+fid, bytes.NewReader(compressedPayload))
+	if err != nil {
+		t.Fatalf("create compressed upload request: %v", err)
+	}
+	uploadReq.Header.Set("Content-Type", "text/plain")
+	uploadReq.Header.Set("Content-Encoding", "gzip")
+	uploadResp := framework.DoRequest(t, client, uploadReq)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("compressed upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	gzipReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	gzipReadReq.Header.Set("Accept-Encoding", "gzip")
+	gzipReadResp := framework.DoRequest(t, client, gzipReadReq)
+	gzipReadBody := framework.ReadAllAndClose(t, gzipReadResp)
+	if gzipReadResp.StatusCode != http.StatusOK {
+		t.Fatalf("gzip-accepted read expected 200, got %d", gzipReadResp.StatusCode)
+	}
+	if gzipReadResp.Header.Get("Content-Encoding") != "gzip" {
+		t.Fatalf("gzip-accepted read expected Content-Encoding=gzip, got %q", gzipReadResp.Header.Get("Content-Encoding"))
+	}
+	if string(gunzipData(t, gzipReadBody)) != string(plainPayload) {
+		t.Fatalf("gzip-accepted read body mismatch after gunzip")
+	}
+
+	identityReadReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	identityReadReq.Header.Set("Accept-Encoding", "identity")
+	identityReadResp := framework.DoRequest(t, client, identityReadReq)
+	identityReadBody := framework.ReadAllAndClose(t, identityReadResp)
+	if identityReadResp.StatusCode != http.StatusOK {
+		t.Fatalf("identity read expected 200, got %d", identityReadResp.StatusCode)
+	}
+	if identityReadResp.Header.Get("Content-Encoding") != "" {
+		t.Fatalf("identity read expected no Content-Encoding header, got %q", identityReadResp.Header.Get("Content-Encoding"))
+	}
+	if string(identityReadBody) != string(plainPayload) {
+		t.Fatalf("identity read body mismatch: got %q want %q", string(identityReadBody), string(plainPayload))
+	}
+}

test/volume_server/http/headers_static_test.go

@@ -0,0 +1,102 @@
+package volume_server_http_test
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+func TestReadPassthroughHeadersAndDownloadDisposition(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(96)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fullFileID := framework.NewFileID(volumeID, 661122, 0x55667788)
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fullFileID, []byte("passthrough-header-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	parts := strings.SplitN(fullFileID, ",", 2)
+	if len(parts) != 2 {
+		t.Fatalf("unexpected file id format: %q", fullFileID)
+	}
+	fidOnly := parts[1]
+
+	url := fmt.Sprintf("%s/%d/%s/%s?response-content-type=text/plain&response-cache-control=no-store&dl=true",
+		clusterHarness.VolumeAdminURL(),
+		volumeID,
+		fidOnly,
+		"report.txt",
+	)
+	resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, url))
+	_ = framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("passthrough read expected 200, got %d", resp.StatusCode)
+	}
+	if resp.Header.Get("Content-Type") != "text/plain" {
+		t.Fatalf("response-content-type override mismatch: %q", resp.Header.Get("Content-Type"))
+	}
+	if resp.Header.Get("Cache-Control") != "no-store" {
+		t.Fatalf("response-cache-control override mismatch: %q", resp.Header.Get("Cache-Control"))
+	}
+	contentDisposition := resp.Header.Get("Content-Disposition")
+	if !strings.Contains(contentDisposition, "attachment") || !strings.Contains(contentDisposition, "report.txt") {
+		t.Fatalf("download disposition header mismatch: %q", contentDisposition)
+	}
+}
+
+func TestStaticAssetEndpoints(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	client := framework.NewHTTPClient()
+
+	faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/favicon.ico"))
+	_ = framework.ReadAllAndClose(t, faviconResp)
+	if faviconResp.StatusCode != http.StatusOK {
+		t.Fatalf("/favicon.ico expected 200, got %d", faviconResp.StatusCode)
+	}
+
+	staticResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/seaweedfsstatic/seaweed50x50.png"))
+	_ = framework.ReadAllAndClose(t, staticResp)
+	if staticResp.StatusCode != http.StatusOK {
+		t.Fatalf("/seaweedfsstatic/seaweed50x50.png expected 200, got %d", staticResp.StatusCode)
+	}
+}
+
+func TestStaticAssetEndpointsOnPublicPort(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	client := framework.NewHTTPClient()
+
+	faviconResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/favicon.ico"))
+	_ = framework.ReadAllAndClose(t, faviconResp)
+	if faviconResp.StatusCode != http.StatusOK {
+		t.Fatalf("public /favicon.ico expected 200, got %d", faviconResp.StatusCode)
+	}
+
+	staticResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/seaweedfsstatic/seaweed50x50.png"))
+	_ = framework.ReadAllAndClose(t, staticResp)
+	if staticResp.StatusCode != http.StatusOK {
+		t.Fatalf("public /seaweedfsstatic/seaweed50x50.png expected 200, got %d", staticResp.StatusCode)
+	}
+}

test/volume_server/http/image_transform_test.go

@@ -0,0 +1,92 @@
+package volume_server_http_test
+
+import (
+	"bytes"
+	"fmt"
+	"image"
+	"image/color"
+	"image/png"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+func makePNGFixture(t testing.TB, width, height int) []byte {
+	t.Helper()
+
+	img := image.NewRGBA(image.Rect(0, 0, width, height))
+	for y := 0; y < height; y++ {
+		for x := 0; x < width; x++ {
+			img.Set(x, y, color.RGBA{R: uint8(x * 20), G: uint8(y * 20), B: 200, A: 255})
+		}
+	}
+
+	var buf bytes.Buffer
+	if err := png.Encode(&buf, img); err != nil {
+		t.Fatalf("encode png fixture: %v", err)
+	}
+	return buf.Bytes()
+}
+
+func decodeImageConfig(t testing.TB, data []byte) image.Config {
+	t.Helper()
+	cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("decode image config: %v", err)
+	}
+	return cfg
+}
+
+func TestImageResizeAndCropReadVariants(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(101)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fullFileID := framework.NewFileID(volumeID, 772004, 0x4D5E6F70)
+	uploadReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fullFileID, makePNGFixture(t, 6, 4))
+	uploadReq.Header.Set("Content-Type", "image/png")
+	uploadResp := framework.DoRequest(t, client, uploadReq)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("image upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	parts := strings.SplitN(fullFileID, ",", 2)
+	if len(parts) != 2 {
+		t.Fatalf("unexpected file id format: %q", fullFileID)
+	}
+	fidOnly := parts[1]
+
+	resizeURL := fmt.Sprintf("%s/%d/%s/%s?width=2&height=1", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "fixture.png")
+	resizeResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, resizeURL))
+	resizeBody := framework.ReadAllAndClose(t, resizeResp)
+	if resizeResp.StatusCode != http.StatusOK {
+		t.Fatalf("image resize read expected 200, got %d", resizeResp.StatusCode)
+	}
+	resizeCfg := decodeImageConfig(t, resizeBody)
+	if resizeCfg.Width > 2 || resizeCfg.Height > 1 {
+		t.Fatalf("image resize expected dimensions <= 2x1, got %dx%d", resizeCfg.Width, resizeCfg.Height)
+	}
+
+	cropURL := fmt.Sprintf("%s/%d/%s/%s?crop_x1=1&crop_y1=1&crop_x2=4&crop_y2=3", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "fixture.png")
+	cropResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cropURL))
+	cropBody := framework.ReadAllAndClose(t, cropResp)
+	if cropResp.StatusCode != http.StatusOK {
+		t.Fatalf("image crop read expected 200, got %d", cropResp.StatusCode)
+	}
+	cropCfg := decodeImageConfig(t, cropBody)
+	if cropCfg.Width != 3 || cropCfg.Height != 2 {
+		t.Fatalf("image crop expected 3x2, got %dx%d", cropCfg.Width, cropCfg.Height)
+	}
+}

test/volume_server/http/public_cors_methods_test.go

@@ -0,0 +1,287 @@
+package volume_server_http_test
+
+import (
+	"bytes"
+	"net/http"
+	"strconv"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+func TestPublicPortReadOnlyMethodBehavior(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(81)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 123321, 0x01020304)
+	originalData := []byte("public-port-original")
+	replacementData := []byte("public-port-replacement")
+	client := framework.NewHTTPClient()
+
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, originalData)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("admin upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	publicReadResp := framework.ReadBytes(t, client, clusterHarness.VolumePublicURL(), fid)
+	publicReadBody := framework.ReadAllAndClose(t, publicReadResp)
+	if publicReadResp.StatusCode != http.StatusOK {
+		t.Fatalf("public GET expected 200, got %d", publicReadResp.StatusCode)
+	}
+	if string(publicReadBody) != string(originalData) {
+		t.Fatalf("public GET body mismatch: got %q want %q", string(publicReadBody), string(originalData))
+	}
+
+	publicPostReq := newUploadRequest(t, clusterHarness.VolumePublicURL()+"/"+fid, replacementData)
+	publicPostResp := framework.DoRequest(t, client, publicPostReq)
+	_ = framework.ReadAllAndClose(t, publicPostResp)
+	if publicPostResp.StatusCode != http.StatusOK {
+		t.Fatalf("public POST expected passthrough 200, got %d", publicPostResp.StatusCode)
+	}
+
+	publicDeleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumePublicURL()+"/"+fid))
+	_ = framework.ReadAllAndClose(t, publicDeleteResp)
+	if publicDeleteResp.StatusCode != http.StatusOK {
+		t.Fatalf("public DELETE expected passthrough 200, got %d", publicDeleteResp.StatusCode)
+	}
+
+	adminReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
+	adminReadBody := framework.ReadAllAndClose(t, adminReadResp)
+	if adminReadResp.StatusCode != http.StatusOK {
+		t.Fatalf("admin GET after public POST/DELETE expected 200, got %d", adminReadResp.StatusCode)
+	}
+	if string(adminReadBody) != string(originalData) {
+		t.Fatalf("public port should not mutate data: got %q want %q", string(adminReadBody), string(originalData))
+	}
+}
+
+func TestCorsAndUnsupportedMethodBehavior(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(82)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 789789, 0x0A0B0C0D)
+	client := framework.NewHTTPClient()
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("cors-check"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("admin upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	adminOriginReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	adminOriginReq.Header.Set("Origin", "https://example.com")
+	adminOriginResp := framework.DoRequest(t, client, adminOriginReq)
+	_ = framework.ReadAllAndClose(t, adminOriginResp)
+	if adminOriginResp.Header.Get("Access-Control-Allow-Origin") != "*" {
+		t.Fatalf("admin GET origin header mismatch: %q", adminOriginResp.Header.Get("Access-Control-Allow-Origin"))
+	}
+	if adminOriginResp.Header.Get("Access-Control-Allow-Credentials") != "true" {
+		t.Fatalf("admin GET credentials header mismatch: %q", adminOriginResp.Header.Get("Access-Control-Allow-Credentials"))
+	}
+
+	publicOriginReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumePublicURL()+"/"+fid)
+	publicOriginReq.Header.Set("Origin", "https://example.com")
+	publicOriginResp := framework.DoRequest(t, client, publicOriginReq)
+	_ = framework.ReadAllAndClose(t, publicOriginResp)
+	if publicOriginResp.Header.Get("Access-Control-Allow-Origin") != "*" {
+		t.Fatalf("public GET origin header mismatch: %q", publicOriginResp.Header.Get("Access-Control-Allow-Origin"))
+	}
+	if publicOriginResp.Header.Get("Access-Control-Allow-Credentials") != "true" {
+		t.Fatalf("public GET credentials header mismatch: %q", publicOriginResp.Header.Get("Access-Control-Allow-Credentials"))
+	}
+
+	adminPatchReq, err := http.NewRequest(http.MethodPatch, clusterHarness.VolumeAdminURL()+"/"+fid, bytes.NewReader([]byte("patch")))
+	if err != nil {
+		t.Fatalf("create admin PATCH request: %v", err)
+	}
+	adminPatchResp := framework.DoRequest(t, client, adminPatchReq)
+	_ = framework.ReadAllAndClose(t, adminPatchResp)
+	if adminPatchResp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("admin PATCH expected 400, got %d", adminPatchResp.StatusCode)
+	}
+
+	publicPatchReq, err := http.NewRequest(http.MethodPatch, clusterHarness.VolumePublicURL()+"/"+fid, bytes.NewReader([]byte("patch")))
+	if err != nil {
+		t.Fatalf("create public PATCH request: %v", err)
+	}
+	publicPatchResp := framework.DoRequest(t, client, publicPatchReq)
+	_ = framework.ReadAllAndClose(t, publicPatchResp)
+	if publicPatchResp.StatusCode != http.StatusOK {
+		t.Fatalf("public PATCH expected passthrough 200, got %d", publicPatchResp.StatusCode)
+	}
+}
+
+func TestUnsupportedMethodTraceParity(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(83)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 123999, 0x01010101)
+	client := framework.NewHTTPClient()
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("trace-method-check"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	adminTraceReq := mustNewRequest(t, http.MethodTrace, clusterHarness.VolumeAdminURL()+"/"+fid)
+	adminTraceResp := framework.DoRequest(t, client, adminTraceReq)
+	_ = framework.ReadAllAndClose(t, adminTraceResp)
+	if adminTraceResp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("admin TRACE expected 400, got %d", adminTraceResp.StatusCode)
+	}
+
+	publicTraceReq := mustNewRequest(t, http.MethodTrace, clusterHarness.VolumePublicURL()+"/"+fid)
+	publicTraceResp := framework.DoRequest(t, client, publicTraceReq)
+	_ = framework.ReadAllAndClose(t, publicTraceResp)
+	if publicTraceResp.StatusCode != http.StatusOK {
+		t.Fatalf("public TRACE expected passthrough 200, got %d", publicTraceResp.StatusCode)
+	}
+}
+
+func TestUnsupportedMethodPropfindParity(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(84)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 124000, 0x02020202)
+	client := framework.NewHTTPClient()
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("propfind-method-check"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	adminReq := mustNewRequest(t, "PROPFIND", clusterHarness.VolumeAdminURL()+"/"+fid)
+	adminResp := framework.DoRequest(t, client, adminReq)
+	_ = framework.ReadAllAndClose(t, adminResp)
+	if adminResp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("admin PROPFIND expected 400, got %d", adminResp.StatusCode)
+	}
+
+	publicReq := mustNewRequest(t, "PROPFIND", clusterHarness.VolumePublicURL()+"/"+fid)
+	publicResp := framework.DoRequest(t, client, publicReq)
+	_ = framework.ReadAllAndClose(t, publicResp)
+	if publicResp.StatusCode != http.StatusOK {
+		t.Fatalf("public PROPFIND expected passthrough 200, got %d", publicResp.StatusCode)
+	}
+
+	verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
+	verifyBody := framework.ReadAllAndClose(t, verifyResp)
+	if verifyResp.StatusCode != http.StatusOK {
+		t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode)
+	}
+	if string(verifyBody) != "propfind-method-check" {
+		t.Fatalf("PROPFIND should not mutate data, got %q", string(verifyBody))
+	}
+}
+
+func TestUnsupportedMethodConnectParity(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(85)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 124001, 0x03030303)
+	client := framework.NewHTTPClient()
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("connect-method-check"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	adminReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumeAdminURL()+"/"+fid)
+	adminResp := framework.DoRequest(t, client, adminReq)
+	_ = framework.ReadAllAndClose(t, adminResp)
+	if adminResp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("admin CONNECT expected 400, got %d", adminResp.StatusCode)
+	}
+
+	publicReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumePublicURL()+"/"+fid)
+	publicResp := framework.DoRequest(t, client, publicReq)
+	_ = framework.ReadAllAndClose(t, publicResp)
+	if publicResp.StatusCode != http.StatusOK {
+		t.Fatalf("public CONNECT expected passthrough 200, got %d", publicResp.StatusCode)
+	}
+
+	verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
+	verifyBody := framework.ReadAllAndClose(t, verifyResp)
+	if verifyResp.StatusCode != http.StatusOK {
+		t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode)
+	}
+	if string(verifyBody) != "connect-method-check" {
+		t.Fatalf("CONNECT should not mutate data, got %q", string(verifyBody))
+	}
+}
+
+func TestPublicPortHeadReadParity(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(86)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 124002, 0x04040404)
+	payload := []byte("public-head-parity-content")
+	client := framework.NewHTTPClient()
+
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumePublicURL()+"/"+fid))
+	headBody := framework.ReadAllAndClose(t, headResp)
+	if headResp.StatusCode != http.StatusOK {
+		t.Fatalf("public HEAD expected 200, got %d", headResp.StatusCode)
+	}
+	if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) {
+		t.Fatalf("public HEAD content-length mismatch: got %q want %d", got, len(payload))
+	}
+	if len(headBody) != 0 {
+		t.Fatalf("public HEAD body should be empty, got %d bytes", len(headBody))
+	}
+}

test/volume_server/http/range_variants_test.go

@@ -0,0 +1,82 @@
+package volume_server_http_test
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+func TestMultiRangeReadReturnsMultipartPayload(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(97)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 771999, 0x0A1B2C3D)
+	payload := []byte("0123456789abcdef")
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	multiRangeReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	multiRangeReq.Header.Set("Range", "bytes=0-1,4-5")
+	multiRangeResp := framework.DoRequest(t, client, multiRangeReq)
+	multiRangeBody := framework.ReadAllAndClose(t, multiRangeResp)
+	if multiRangeResp.StatusCode != http.StatusPartialContent {
+		t.Fatalf("multi-range expected 206, got %d", multiRangeResp.StatusCode)
+	}
+	if !strings.Contains(multiRangeResp.Header.Get("Content-Type"), "multipart/byteranges") {
+		t.Fatalf("multi-range content-type mismatch: %q", multiRangeResp.Header.Get("Content-Type"))
+	}
+
+	bodyText := string(multiRangeBody)
+	if !strings.Contains(bodyText, "01") || !strings.Contains(bodyText, "45") {
+		t.Fatalf("multi-range body missing expected segments: %q", bodyText)
+	}
+}
+
+func TestOversizedCombinedRangesAreIgnored(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(100)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 772003, 0x3C4D5E6F)
+	payload := []byte("0123456789abcdef")
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	// Range bytes sum is 22 (> payload size 16), which exercises the oversized-range guard path.
+	oversizedRangeReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	oversizedRangeReq.Header.Set("Range", "bytes=0-10,5-15")
+	oversizedRangeResp := framework.DoRequest(t, client, oversizedRangeReq)
+	oversizedRangeBody := framework.ReadAllAndClose(t, oversizedRangeResp)
+	if oversizedRangeResp.StatusCode != http.StatusOK {
+		t.Fatalf("oversized combined range expected 200, got %d", oversizedRangeResp.StatusCode)
+	}
+	if len(oversizedRangeBody) != 0 {
+		t.Fatalf("oversized combined range expected empty body, got %d bytes", len(oversizedRangeBody))
+	}
+}

test/volume_server/http/read_deleted_test.go

@@ -0,0 +1,54 @@
+package volume_server_http_test
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+func TestReadDeletedQueryReturnsDeletedNeedleData(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(94)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 551234, 0xCAFE1234)
+	payload := []byte("read-deleted-needle-payload")
+
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid))
+	_ = framework.ReadAllAndClose(t, deleteResp)
+	if deleteResp.StatusCode != http.StatusAccepted {
+		t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode)
+	}
+
+	normalRead := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
+	_ = framework.ReadAllAndClose(t, normalRead)
+	if normalRead.StatusCode != http.StatusNotFound {
+		t.Fatalf("normal read after delete expected 404, got %d", normalRead.StatusCode)
+	}
+
+	readDeletedReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true")
+	readDeletedResp := framework.DoRequest(t, client, readDeletedReq)
+	readDeletedBody := framework.ReadAllAndClose(t, readDeletedResp)
+	if readDeletedResp.StatusCode != http.StatusOK {
+		t.Fatalf("read with readDeleted=true expected 200, got %d", readDeletedResp.StatusCode)
+	}
+	if string(readDeletedBody) != string(payload) {
+		t.Fatalf("readDeleted body mismatch: got %q want %q", string(readDeletedBody), string(payload))
+	}
+}

test/volume_server/http/read_mode_proxy_redirect_test.go

@@ -0,0 +1,319 @@
+package volume_server_http_test
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+func TestReadModeProxyMissingLocalVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P1()
+	profile.ReadMode = "proxy"
+	clusterHarness := framework.StartDualVolumeCluster(t, profile)
+
+	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer conn0.Close()
+
+	const volumeID = uint32(101)
+	framework.AllocateVolume(t, grpc0, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 120001, 0x0102ABCD)
+	payload := []byte("proxy-read-mode-forwarded-content")
+
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid
+	var finalBody []byte
+	if !waitForHTTPStatus(t, client, readURL, http.StatusOK, 10*time.Second, func(resp *http.Response) {
+		finalBody = framework.ReadAllAndClose(t, resp)
+	}) {
+		t.Fatalf("proxy read mode did not return 200 from non-owning volume server within deadline")
+	}
+	if string(finalBody) != string(payload) {
+		t.Fatalf("proxy read mode body mismatch: got %q want %q", string(finalBody), string(payload))
+	}
+}
+
+func TestReadModeRedirectMissingLocalVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P1()
+	profile.ReadMode = "redirect"
+	clusterHarness := framework.StartDualVolumeCluster(t, profile)
+
+	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer conn0.Close()
+
+	const volumeID = uint32(102)
+	framework.AllocateVolume(t, grpc0, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 120002, 0x0102DCBA)
+	payload := []byte("redirect-read-mode-content")
+
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	noRedirectClient := &http.Client{
+		Timeout: 10 * time.Second,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid
+	var redirectLocation string
+	if !waitForHTTPStatus(t, noRedirectClient, readURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) {
+		redirectLocation = resp.Header.Get("Location")
+		_ = framework.ReadAllAndClose(t, resp)
+	}) {
+		t.Fatalf("redirect read mode did not return 301 from non-owning volume server within deadline")
+	}
+	if redirectLocation == "" {
+		t.Fatalf("redirect response missing Location header")
+	}
+	if !strings.Contains(redirectLocation, "proxied=true") {
+		t.Fatalf("redirect Location should include proxied=true, got %q", redirectLocation)
+	}
+
+	followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, redirectLocation))
+	followBody := framework.ReadAllAndClose(t, followResp)
+	if followResp.StatusCode != http.StatusOK {
+		t.Fatalf("following redirect expected 200, got %d", followResp.StatusCode)
+	}
+	if string(followBody) != string(payload) {
+		t.Fatalf("redirect-follow body mismatch: got %q want %q", string(followBody), string(payload))
+	}
+}
+
+func TestReadModeLocalMissingLocalVolumeReturnsNotFound(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P1()
+	profile.ReadMode = "local"
+	clusterHarness := framework.StartDualVolumeCluster(t, profile)
+
+	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer conn0.Close()
+
+	const volumeID = uint32(103)
+	framework.AllocateVolume(t, grpc0, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 120003, 0x0102BEEF)
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, []byte("local-read-mode-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	readResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(1), fid)
+	_ = framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusNotFound {
+		t.Fatalf("local read mode expected 404 on non-owning server, got %d", readResp.StatusCode)
+	}
+}
+
+func TestReadDeletedProxyModeOnMissingLocalVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P1()
+	profile.ReadMode = "proxy"
+	clusterHarness := framework.StartDualVolumeCluster(t, profile)
+
+	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer conn0.Close()
+
+	const volumeID = uint32(104)
+	framework.AllocateVolume(t, grpc0, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 120004, 0x0102CAFE)
+	payload := []byte("proxy-readDeleted-missing-local-content")
+
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL(0)+"/"+fid))
+	_ = framework.ReadAllAndClose(t, deleteResp)
+	if deleteResp.StatusCode != http.StatusAccepted {
+		t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode)
+	}
+
+	readURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?readDeleted=true"
+	var proxiedBody []byte
+	if !waitForHTTPStatus(t, client, readURL, http.StatusOK, 10*time.Second, func(resp *http.Response) {
+		proxiedBody = framework.ReadAllAndClose(t, resp)
+	}) {
+		t.Fatalf("proxy readDeleted path did not return 200 from non-owning volume server within deadline")
+	}
+	if string(proxiedBody) != string(payload) {
+		t.Fatalf("proxy readDeleted body mismatch: got %q want %q", string(proxiedBody), string(payload))
+	}
+}
+
+func TestReadDeletedRedirectModeDropsQueryParameterParity(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P1()
+	profile.ReadMode = "redirect"
+	clusterHarness := framework.StartDualVolumeCluster(t, profile)
+
+	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer conn0.Close()
+
+	const volumeID = uint32(105)
+	framework.AllocateVolume(t, grpc0, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 120005, 0x0102FACE)
+	payload := []byte("redirect-readDeleted-query-drop-parity")
+
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL(0)+"/"+fid))
+	_ = framework.ReadAllAndClose(t, deleteResp)
+	if deleteResp.StatusCode != http.StatusAccepted {
+		t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode)
+	}
+
+	noRedirectClient := &http.Client{
+		Timeout: 10 * time.Second,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	redirectURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?readDeleted=true"
+	var location string
+	if !waitForHTTPStatus(t, noRedirectClient, redirectURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) {
+		location = resp.Header.Get("Location")
+		_ = framework.ReadAllAndClose(t, resp)
+	}) {
+		t.Fatalf("redirect readDeleted path did not return 301 from non-owning volume server within deadline")
+	}
+	if location == "" {
+		t.Fatalf("redirect readDeleted response missing Location header")
+	}
+	if !strings.Contains(location, "proxied=true") {
+		t.Fatalf("redirect readDeleted Location should include proxied=true, got %q", location)
+	}
+	if strings.Contains(location, "readDeleted=true") {
+		t.Fatalf("redirect readDeleted Location should reflect current query-drop behavior, got %q", location)
+	}
+
+	followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, location))
+	_ = framework.ReadAllAndClose(t, followResp)
+	if followResp.StatusCode != http.StatusNotFound {
+		t.Fatalf("redirect-follow without readDeleted query expected 404 for deleted needle, got %d", followResp.StatusCode)
+	}
+}
+
+func TestReadModeRedirectPreservesCollectionQuery(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P1()
+	profile.ReadMode = "redirect"
+	clusterHarness := framework.StartDualVolumeCluster(t, profile)
+
+	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer conn0.Close()
+
+	const volumeID = uint32(109)
+	const collection = "redirect-collection"
+	framework.AllocateVolume(t, grpc0, volumeID, collection)
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 120006, 0x0102F00D)
+	payload := []byte("redirect-collection-preserve-content")
+
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(0), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	noRedirectClient := &http.Client{
+		Timeout: 10 * time.Second,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	redirectURL := clusterHarness.VolumeAdminURL(1) + "/" + fid + "?collection=" + collection
+	var location string
+	if !waitForHTTPStatus(t, noRedirectClient, redirectURL, http.StatusMovedPermanently, 10*time.Second, func(resp *http.Response) {
+		location = resp.Header.Get("Location")
+		_ = framework.ReadAllAndClose(t, resp)
+	}) {
+		t.Fatalf("redirect collection path did not return 301 from non-owning volume server within deadline")
+	}
+	if location == "" {
+		t.Fatalf("redirect collection response missing Location header")
+	}
+	if !strings.Contains(location, "proxied=true") {
+		t.Fatalf("redirect collection Location should include proxied=true, got %q", location)
+	}
+	if !strings.Contains(location, "collection="+collection) {
+		t.Fatalf("redirect collection Location should preserve collection query, got %q", location)
+	}
+
+	followResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, location))
+	followBody := framework.ReadAllAndClose(t, followResp)
+	if followResp.StatusCode != http.StatusOK {
+		t.Fatalf("redirect-follow expected 200, got %d", followResp.StatusCode)
+	}
+	if string(followBody) != string(payload) {
+		t.Fatalf("redirect-follow body mismatch: got %q want %q", string(followBody), string(payload))
+	}
+}
+
+func waitForHTTPStatus(t testing.TB, client *http.Client, url string, expectedStatus int, timeout time.Duration, onMatch func(resp *http.Response)) bool {
+	t.Helper()
+
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, url))
+		if resp.StatusCode == expectedStatus {
+			onMatch(resp)
+			return true
+		}
+		_ = framework.ReadAllAndClose(t, resp)
+		time.Sleep(200 * time.Millisecond)
+	}
+
+	return false
+}

test/volume_server/http/read_path_variants_test.go

@@ -0,0 +1,191 @@
+package volume_server_http_test
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+func TestReadPathShapesAndIfModifiedSince(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(93)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fullFileID := framework.NewFileID(volumeID, 771234, 0xBEEFCACE)
+	uploadPayload := []byte("read-path-shape-content")
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fullFileID, uploadPayload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	parts := strings.SplitN(fullFileID, ",", 2)
+	if len(parts) != 2 {
+		t.Fatalf("unexpected file id format: %q", fullFileID)
+	}
+	fidOnly := parts[1]
+
+	readByVidFid := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, fmt.Sprintf("%s/%d/%s", clusterHarness.VolumeAdminURL(), volumeID, fidOnly)))
+	readByVidFidBody := framework.ReadAllAndClose(t, readByVidFid)
+	if readByVidFid.StatusCode != http.StatusOK {
+		t.Fatalf("GET /{vid}/{fid} expected 200, got %d", readByVidFid.StatusCode)
+	}
+	if string(readByVidFidBody) != string(uploadPayload) {
+		t.Fatalf("GET /{vid}/{fid} body mismatch: got %q want %q", string(readByVidFidBody), string(uploadPayload))
+	}
+
+	readWithFilename := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, fmt.Sprintf("%s/%d/%s/%s", clusterHarness.VolumeAdminURL(), volumeID, fidOnly, "named.bin")))
+	readWithFilenameBody := framework.ReadAllAndClose(t, readWithFilename)
+	if readWithFilename.StatusCode != http.StatusOK {
+		t.Fatalf("GET /{vid}/{fid}/{filename} expected 200, got %d", readWithFilename.StatusCode)
+	}
+	if string(readWithFilenameBody) != string(uploadPayload) {
+		t.Fatalf("GET /{vid}/{fid}/{filename} body mismatch: got %q want %q", string(readWithFilenameBody), string(uploadPayload))
+	}
+
+	lastModified := readWithFilename.Header.Get("Last-Modified")
+	if lastModified == "" {
+		t.Fatalf("expected Last-Modified header on read response")
+	}
+
+	ifModifiedSinceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fullFileID)
+	ifModifiedSinceReq.Header.Set("If-Modified-Since", lastModified)
+	ifModifiedSinceResp := framework.DoRequest(t, client, ifModifiedSinceReq)
+	_ = framework.ReadAllAndClose(t, ifModifiedSinceResp)
+	if ifModifiedSinceResp.StatusCode != http.StatusNotModified {
+		t.Fatalf("If-Modified-Since expected 304, got %d", ifModifiedSinceResp.StatusCode)
+	}
+
+	headIfModifiedSinceReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fullFileID)
+	headIfModifiedSinceReq.Header.Set("If-Modified-Since", lastModified)
+	headIfModifiedSinceResp := framework.DoRequest(t, client, headIfModifiedSinceReq)
+	headIfModifiedSinceBody := framework.ReadAllAndClose(t, headIfModifiedSinceResp)
+	if headIfModifiedSinceResp.StatusCode != http.StatusNotModified {
+		t.Fatalf("HEAD If-Modified-Since expected 304, got %d", headIfModifiedSinceResp.StatusCode)
+	}
+	if len(headIfModifiedSinceBody) != 0 {
+		t.Fatalf("HEAD If-Modified-Since expected empty body, got %d bytes", len(headIfModifiedSinceBody))
+	}
+}
+
+func TestMalformedVidFidPathReturnsBadRequest(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	client := framework.NewHTTPClient()
+
+	resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/not-a-vid/not-a-fid"))
+	_ = framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("malformed /{vid}/{fid} expected 400, got %d", resp.StatusCode)
+	}
+}
+
+func TestReadWrongCookieReturnsNotFound(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(95)
+	const needleID = uint64(771235)
+	const cookie = uint32(0xBEEFCACF)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("read-cookie-mismatch-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	wrongCookieFid := framework.NewFileID(volumeID, needleID, cookie+1)
+	getResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), wrongCookieFid)
+	_ = framework.ReadAllAndClose(t, getResp)
+	if getResp.StatusCode != http.StatusNotFound {
+		t.Fatalf("GET with wrong cookie expected 404, got %d", getResp.StatusCode)
+	}
+
+	headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+wrongCookieFid))
+	headBody := framework.ReadAllAndClose(t, headResp)
+	if headResp.StatusCode != http.StatusNotFound {
+		t.Fatalf("HEAD with wrong cookie expected 404, got %d", headResp.StatusCode)
+	}
+	if len(headBody) != 0 {
+		t.Fatalf("HEAD wrong-cookie response body should be empty, got %d bytes", len(headBody))
+	}
+}
+
+func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(99)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 772002, 0x2B3C4D5E)
+	payload := []byte("conditional-precedence-content")
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	baselineResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
+	_ = framework.ReadAllAndClose(t, baselineResp)
+	if baselineResp.StatusCode != http.StatusOK {
+		t.Fatalf("baseline read expected 200, got %d", baselineResp.StatusCode)
+	}
+	lastModified := baselineResp.Header.Get("Last-Modified")
+	if lastModified == "" {
+		t.Fatalf("baseline read expected Last-Modified header")
+	}
+
+	precedenceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	precedenceReq.Header.Set("If-Modified-Since", lastModified)
+	precedenceReq.Header.Set("If-None-Match", "\"definitely-different-etag\"")
+	precedenceResp := framework.DoRequest(t, client, precedenceReq)
+	precedenceBody := framework.ReadAllAndClose(t, precedenceResp)
+	if precedenceResp.StatusCode != http.StatusNotModified {
+		t.Fatalf("conditional precedence expected 304, got %d", precedenceResp.StatusCode)
+	}
+	if len(precedenceBody) != 0 {
+		t.Fatalf("conditional precedence expected empty body, got %d bytes", len(precedenceBody))
+	}
+
+	invalidIMSReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
+	invalidIMSReq.Header.Set("If-Modified-Since", "not-a-valid-http-date")
+	invalidIMSReq.Header.Set("If-None-Match", "\"definitely-different-etag\"")
+	invalidIMSResp := framework.DoRequest(t, client, invalidIMSReq)
+	invalidIMSBody := framework.ReadAllAndClose(t, invalidIMSResp)
+	if invalidIMSResp.StatusCode != http.StatusOK {
+		t.Fatalf("invalid If-Modified-Since with mismatched etag expected 200, got %d", invalidIMSResp.StatusCode)
+	}
+	if string(invalidIMSBody) != string(payload) {
+		t.Fatalf("invalid If-Modified-Since fallback body mismatch: got %q want %q", string(invalidIMSBody), string(payload))
+	}
+}

test/volume_server/http/read_write_delete_test.go

@@ -0,0 +1,123 @@
+package volume_server_http_test
+
+import (
+	"net/http"
+	"strconv"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+func TestUploadReadRangeHeadDeleteRoundTrip(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(7)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 123456, 0xA1B2C3D4)
+	data := []byte("hello-volume-server-integration")
+	client := framework.NewHTTPClient()
+
+	uploadResp := framework.UploadBytes(t, client, cluster.VolumeAdminURL(), fid, data)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	getResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
+	getBody := framework.ReadAllAndClose(t, getResp)
+	if getResp.StatusCode != http.StatusOK {
+		t.Fatalf("get status: expected 200, got %d", getResp.StatusCode)
+	}
+	if string(getBody) != string(data) {
+		t.Fatalf("get body mismatch: got %q want %q", string(getBody), string(data))
+	}
+	etag := getResp.Header.Get("ETag")
+	if etag == "" {
+		t.Fatalf("expected ETag header from GET response")
+	}
+
+	notModifiedReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid)
+	notModifiedReq.Header.Set("If-None-Match", etag)
+	notModifiedResp := framework.DoRequest(t, client, notModifiedReq)
+	_ = framework.ReadAllAndClose(t, notModifiedResp)
+	if notModifiedResp.StatusCode != http.StatusNotModified {
+		t.Fatalf("if-none-match expected 304, got %d", notModifiedResp.StatusCode)
+	}
+
+	rangeReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid)
+	rangeReq.Header.Set("Range", "bytes=0-4")
+	rangeResp := framework.DoRequest(t, client, rangeReq)
+	rangeBody := framework.ReadAllAndClose(t, rangeResp)
+	if rangeResp.StatusCode != http.StatusPartialContent {
+		t.Fatalf("range status: expected 206, got %d", rangeResp.StatusCode)
+	}
+	if got, want := string(rangeBody), "hello"; got != want {
+		t.Fatalf("range body mismatch: got %q want %q", got, want)
+	}
+
+	invalidRangeReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/"+fid)
+	invalidRangeReq.Header.Set("Range", "bytes=9999-10000")
+	invalidRangeResp := framework.DoRequest(t, client, invalidRangeReq)
+	_ = framework.ReadAllAndClose(t, invalidRangeResp)
+	if invalidRangeResp.StatusCode != http.StatusRequestedRangeNotSatisfiable {
+		t.Fatalf("invalid range expected 416, got %d", invalidRangeResp.StatusCode)
+	}
+
+	headResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodHead, cluster.VolumeAdminURL()+"/"+fid))
+	headBody := framework.ReadAllAndClose(t, headResp)
+	if headResp.StatusCode != http.StatusOK {
+		t.Fatalf("head status: expected 200, got %d", headResp.StatusCode)
+	}
+	if got := headResp.Header.Get("Content-Length"); got != strconv.Itoa(len(data)) {
+		t.Fatalf("head content-length mismatch: got %q want %d", got, len(data))
+	}
+	if len(headBody) != 0 {
+		t.Fatalf("head body should be empty, got %d bytes", len(headBody))
+	}
+
+	headNotModifiedReq := mustNewRequest(t, http.MethodHead, cluster.VolumeAdminURL()+"/"+fid)
+	headNotModifiedReq.Header.Set("If-None-Match", etag)
+	headNotModifiedResp := framework.DoRequest(t, client, headNotModifiedReq)
+	headNotModifiedBody := framework.ReadAllAndClose(t, headNotModifiedResp)
+	if headNotModifiedResp.StatusCode != http.StatusNotModified {
+		t.Fatalf("head if-none-match expected 304, got %d", headNotModifiedResp.StatusCode)
+	}
+	if len(headNotModifiedBody) != 0 {
+		t.Fatalf("head if-none-match body should be empty, got %d bytes", len(headNotModifiedBody))
+	}
+
+	deleteResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, cluster.VolumeAdminURL()+"/"+fid))
+	_ = framework.ReadAllAndClose(t, deleteResp)
+	if deleteResp.StatusCode != http.StatusAccepted {
+		t.Fatalf("delete status: expected 202, got %d", deleteResp.StatusCode)
+	}
+
+	notFoundResp := framework.ReadBytes(t, client, cluster.VolumeAdminURL(), fid)
+	_ = framework.ReadAllAndClose(t, notFoundResp)
+	if notFoundResp.StatusCode != http.StatusNotFound {
+		t.Fatalf("read after delete: expected 404, got %d", notFoundResp.StatusCode)
+	}
+}
+
+func TestInvalidReadPathReturnsBadRequest(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	client := framework.NewHTTPClient()
+
+	resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/invalid,needle"))
+	_ = framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("invalid read expected 400, got %d", resp.StatusCode)
+	}
+}

test/volume_server/http/throttling_test.go

@@ -0,0 +1,730 @@
+package volume_server_http_test
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+type pausableReader struct {
+	remaining  int64
+	pauseAfter int64
+	paused     bool
+	unblock    <-chan struct{}
+}
+
+func (r *pausableReader) Read(p []byte) (int, error) {
+	if r.remaining <= 0 {
+		return 0, io.EOF
+	}
+	if !r.paused && r.pauseAfter > 0 {
+		n := int64(len(p))
+		if n > r.pauseAfter {
+			n = r.pauseAfter
+		}
+		for i := int64(0); i < n; i++ {
+			p[i] = 'a'
+		}
+		r.remaining -= n
+		r.pauseAfter -= n
+		if r.pauseAfter == 0 {
+			r.paused = true
+		}
+		return int(n), nil
+	}
+	if r.paused {
+		<-r.unblock
+		r.paused = false
+	}
+	n := int64(len(p))
+	if n > r.remaining {
+		n = r.remaining
+	}
+	for i := int64(0); i < n; i++ {
+		p[i] = 'b'
+	}
+	r.remaining -= n
+	return int(n), nil
+}
+
+func TestUploadLimitTimeoutAndReplicateBypass(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(98)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	const blockedUploadSize = 2 * 1024 * 1024 // over 1MB P8 upload limit
+
+	unblockFirstUpload := make(chan struct{})
+	firstUploadDone := make(chan error, 1)
+	firstFID := framework.NewFileID(volumeID, 880001, 0x1A2B3C4D)
+	go func() {
+		req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{
+			remaining:  blockedUploadSize,
+			pauseAfter: 1,
+			unblock:    unblockFirstUpload,
+		})
+		if err != nil {
+			firstUploadDone <- err
+			return
+		}
+		req.Header.Set("Content-Type", "application/octet-stream")
+		req.ContentLength = blockedUploadSize
+
+		resp, err := (&http.Client{}).Do(req)
+		if resp != nil {
+			_, _ = io.Copy(io.Discard, resp.Body)
+			_ = resp.Body.Close()
+		}
+		firstUploadDone <- err
+	}()
+
+	// Give the first upload time to pass limit checks and block in body processing.
+	time.Sleep(300 * time.Millisecond)
+
+	replicateFID := framework.NewFileID(volumeID, 880002, 0x5E6F7A8B)
+	replicateReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+replicateFID+"?type=replicate", bytes.NewReader([]byte("replicate")))
+	if err != nil {
+		t.Fatalf("create replicate request: %v", err)
+	}
+	replicateReq.Header.Set("Content-Type", "application/octet-stream")
+	replicateReq.ContentLength = int64(len("replicate"))
+	replicateResp, err := framework.NewHTTPClient().Do(replicateReq)
+	if err != nil {
+		t.Fatalf("replicate request failed: %v", err)
+	}
+	_ = framework.ReadAllAndClose(t, replicateResp)
+	if replicateResp.StatusCode != http.StatusCreated {
+		t.Fatalf("replicate request expected 201 bypassing limit, got %d", replicateResp.StatusCode)
+	}
+
+	normalFID := framework.NewFileID(volumeID, 880003, 0x9C0D1E2F)
+	normalReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+normalFID, bytes.NewReader([]byte("normal")))
+	if err != nil {
+		t.Fatalf("create normal request: %v", err)
+	}
+	normalReq.Header.Set("Content-Type", "application/octet-stream")
+	normalReq.ContentLength = int64(len("normal"))
+
+	timeoutClient := &http.Client{Timeout: 10 * time.Second}
+	normalResp, err := timeoutClient.Do(normalReq)
+	if err != nil {
+		t.Fatalf("normal upload request failed: %v", err)
+	}
+	_ = framework.ReadAllAndClose(t, normalResp)
+	if normalResp.StatusCode != http.StatusTooManyRequests {
+		t.Fatalf("normal upload expected 429 while limit blocked, got %d", normalResp.StatusCode)
+	}
+
+	close(unblockFirstUpload)
+	select {
+	case <-firstUploadDone:
+	case <-time.After(5 * time.Second):
+		t.Fatalf("timed out waiting for blocked upload to finish")
+	}
+}
+
+func TestUploadLimitWaitThenProceed(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(111)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	const blockedUploadSize = 2 * 1024 * 1024
+
+	unblockFirstUpload := make(chan struct{})
+	firstUploadDone := make(chan error, 1)
+	firstFID := framework.NewFileID(volumeID, 880601, 0x6A2B3C4D)
+	go func() {
+		req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{
+			remaining:  blockedUploadSize,
+			pauseAfter: 1,
+			unblock:    unblockFirstUpload,
+		})
+		if err != nil {
+			firstUploadDone <- err
+			return
+		}
+		req.Header.Set("Content-Type", "application/octet-stream")
+		req.ContentLength = blockedUploadSize
+
+		resp, err := (&http.Client{}).Do(req)
+		if resp != nil {
+			_ = framework.ReadAllAndClose(t, resp)
+		}
+		firstUploadDone <- err
+	}()
+
+	time.Sleep(300 * time.Millisecond)
+
+	type uploadResult struct {
+		resp *http.Response
+		err  error
+	}
+	secondUploadDone := make(chan uploadResult, 1)
+	secondFID := framework.NewFileID(volumeID, 880602, 0x6A2B3C4E)
+	go func() {
+		req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+secondFID, bytes.NewReader([]byte("wait-then-proceed")))
+		if err != nil {
+			secondUploadDone <- uploadResult{err: err}
+			return
+		}
+		req.Header.Set("Content-Type", "application/octet-stream")
+		req.ContentLength = int64(len("wait-then-proceed"))
+		resp, err := (&http.Client{Timeout: 10 * time.Second}).Do(req)
+		secondUploadDone <- uploadResult{resp: resp, err: err}
+	}()
+
+	time.Sleep(500 * time.Millisecond)
+	close(unblockFirstUpload)
+
+	select {
+	case firstErr := <-firstUploadDone:
+		if firstErr != nil {
+			t.Fatalf("first blocked upload failed: %v", firstErr)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatalf("timed out waiting for first upload completion")
+	}
+
+	select {
+	case result := <-secondUploadDone:
+		if result.err != nil {
+			t.Fatalf("second upload failed: %v", result.err)
+		}
+		_ = framework.ReadAllAndClose(t, result.resp)
+		if result.resp.StatusCode != http.StatusCreated {
+			t.Fatalf("second upload expected 201 after waiting for slot, got %d", result.resp.StatusCode)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatalf("timed out waiting for second upload completion")
+	}
+}
+
+func TestUploadLimitTimeoutThenRecovery(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(113)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	const blockedUploadSize = 2 * 1024 * 1024
+
+	unblockFirstUpload := make(chan struct{})
+	firstUploadDone := make(chan error, 1)
+	firstFID := framework.NewFileID(volumeID, 880801, 0x7A2B3C4D)
+	go func() {
+		req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{
+			remaining:  blockedUploadSize,
+			pauseAfter: 1,
+			unblock:    unblockFirstUpload,
+		})
+		if err != nil {
+			firstUploadDone <- err
+			return
+		}
+		req.Header.Set("Content-Type", "application/octet-stream")
+		req.ContentLength = blockedUploadSize
+		resp, err := (&http.Client{}).Do(req)
+		if resp != nil {
+			_ = framework.ReadAllAndClose(t, resp)
+		}
+		firstUploadDone <- err
+	}()
+
+	time.Sleep(300 * time.Millisecond)
+
+	timeoutFID := framework.NewFileID(volumeID, 880802, 0x7A2B3C4E)
+	timeoutResp := framework.UploadBytes(t, &http.Client{Timeout: 10 * time.Second}, clusterHarness.VolumeAdminURL(), timeoutFID, []byte("should-timeout"))
+	_ = framework.ReadAllAndClose(t, timeoutResp)
+	if timeoutResp.StatusCode != http.StatusTooManyRequests {
+		t.Fatalf("second upload under blocked pressure expected 429, got %d", timeoutResp.StatusCode)
+	}
+
+	close(unblockFirstUpload)
+	select {
+	case firstErr := <-firstUploadDone:
+		if firstErr != nil {
+			t.Fatalf("first blocked upload failed: %v", firstErr)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatalf("timed out waiting for first upload completion")
+	}
+
+	recoveryFID := framework.NewFileID(volumeID, 880803, 0x7A2B3C4F)
+	recoveryResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), recoveryFID, []byte("recovered-upload"))
+	_ = framework.ReadAllAndClose(t, recoveryResp)
+	if recoveryResp.StatusCode != http.StatusCreated {
+		t.Fatalf("recovery upload expected 201, got %d", recoveryResp.StatusCode)
+	}
+}
+
+func TestDownloadLimitTimeoutReturnsTooManyRequests(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(99)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	largePayload := make([]byte, 12*1024*1024) // over 1MB P8 download limit
+	for i := range largePayload {
+		largePayload[i] = byte(i % 251)
+	}
+	downloadFID := framework.NewFileID(volumeID, 880101, 0x10203040)
+	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), downloadFID, largePayload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+downloadFID))
+	if err != nil {
+		t.Fatalf("first GET failed: %v", err)
+	}
+	if firstResp.StatusCode != http.StatusOK {
+		_ = framework.ReadAllAndClose(t, firstResp)
+		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
+	}
+	defer firstResp.Body.Close()
+
+	// Keep first response body unread so server write path stays in-flight.
+	time.Sleep(300 * time.Millisecond)
+
+	secondClient := &http.Client{Timeout: 10 * time.Second}
+	secondResp, err := secondClient.Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+downloadFID))
+	if err != nil {
+		t.Fatalf("second GET failed: %v", err)
+	}
+	_ = framework.ReadAllAndClose(t, secondResp)
+	if secondResp.StatusCode != http.StatusTooManyRequests {
+		t.Fatalf("second GET expected 429 while first download holds limit, got %d", secondResp.StatusCode)
+	}
+}
+
+func TestDownloadLimitWaitThenProceedWithoutReplica(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(112)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	largePayload := make([]byte, 12*1024*1024)
+	for i := range largePayload {
+		largePayload[i] = byte(i % 251)
+	}
+	fid := framework.NewFileID(volumeID, 880701, 0x60708090)
+	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid))
+	if err != nil {
+		t.Fatalf("first GET failed: %v", err)
+	}
+	if firstResp.StatusCode != http.StatusOK {
+		_ = framework.ReadAllAndClose(t, firstResp)
+		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
+	}
+
+	type readResult struct {
+		resp *http.Response
+		err  error
+	}
+	secondReadDone := make(chan readResult, 1)
+	go func() {
+		resp, readErr := (&http.Client{Timeout: 10 * time.Second}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid))
+		secondReadDone <- readResult{resp: resp, err: readErr}
+	}()
+
+	time.Sleep(500 * time.Millisecond)
+	_ = firstResp.Body.Close()
+
+	select {
+	case result := <-secondReadDone:
+		if result.err != nil {
+			t.Fatalf("second GET failed: %v", result.err)
+		}
+		secondBody := framework.ReadAllAndClose(t, result.resp)
+		if result.resp.StatusCode != http.StatusOK {
+			t.Fatalf("second GET expected 200 after waiting for slot, got %d", result.resp.StatusCode)
+		}
+		if len(secondBody) != len(largePayload) {
+			t.Fatalf("second GET body size mismatch: got %d want %d", len(secondBody), len(largePayload))
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatalf("timed out waiting for second GET completion")
+	}
+}
+
+func TestDownloadLimitTimeoutThenRecovery(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(114)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	largePayload := make([]byte, 12*1024*1024)
+	for i := range largePayload {
+		largePayload[i] = byte(i % 251)
+	}
+	fid := framework.NewFileID(volumeID, 880901, 0x708090A0)
+	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid))
+	if err != nil {
+		t.Fatalf("first GET failed: %v", err)
+	}
+	if firstResp.StatusCode != http.StatusOK {
+		_ = framework.ReadAllAndClose(t, firstResp)
+		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
+	}
+
+	time.Sleep(300 * time.Millisecond)
+
+	timeoutResp := framework.ReadBytes(t, &http.Client{Timeout: 10 * time.Second}, clusterHarness.VolumeAdminURL(), fid)
+	_ = framework.ReadAllAndClose(t, timeoutResp)
+	if timeoutResp.StatusCode != http.StatusTooManyRequests {
+		t.Fatalf("second GET under blocked pressure expected 429, got %d", timeoutResp.StatusCode)
+	}
+
+	_ = firstResp.Body.Close()
+
+	recoveryResp := framework.ReadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid)
+	recoveryBody := framework.ReadAllAndClose(t, recoveryResp)
+	if recoveryResp.StatusCode != http.StatusOK {
+		t.Fatalf("recovery GET expected 200, got %d", recoveryResp.StatusCode)
+	}
+	if len(recoveryBody) != len(largePayload) {
+		t.Fatalf("recovery GET body size mismatch: got %d want %d", len(recoveryBody), len(largePayload))
+	}
+}
+
+func TestDownloadLimitOverageProxiesToReplica(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P8()
+	profile.ReadMode = "proxy"
+	clusterHarness := framework.StartDualVolumeCluster(t, profile)
+
+	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, grpc1 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	const volumeID = uint32(100)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	req := &volume_server_pb.AllocateVolumeRequest{
+		VolumeId:    volumeID,
+		Replication: "001",
+		Version:     uint32(needle.GetCurrentVersion()),
+	}
+	if _, err := grpc0.AllocateVolume(ctx, req); err != nil {
+		t.Fatalf("allocate replicated volume on node0: %v", err)
+	}
+	if _, err := grpc1.AllocateVolume(ctx, req); err != nil {
+		t.Fatalf("allocate replicated volume on node1: %v", err)
+	}
+
+	largePayload := make([]byte, 12*1024*1024)
+	for i := range largePayload {
+		largePayload[i] = byte(i % 251)
+	}
+	fid := framework.NewFileID(volumeID, 880201, 0x0A0B0C0D)
+	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(0), fid, largePayload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("replicated large upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	replicaReadURL := clusterHarness.VolumeAdminURL(1) + "/" + fid
+	if !waitForHTTPStatus(t, framework.NewHTTPClient(), replicaReadURL, http.StatusOK, 10*time.Second, func(resp *http.Response) {
+		_ = framework.ReadAllAndClose(t, resp)
+	}) {
+		t.Fatalf("replica did not become readable within deadline")
+	}
+
+	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid))
+	if err != nil {
+		t.Fatalf("first GET failed: %v", err)
+	}
+	if firstResp.StatusCode != http.StatusOK {
+		_ = framework.ReadAllAndClose(t, firstResp)
+		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
+	}
+	defer firstResp.Body.Close()
+
+	time.Sleep(300 * time.Millisecond)
+
+	secondResp, err := framework.NewHTTPClient().Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid))
+	if err != nil {
+		t.Fatalf("second GET failed: %v", err)
+	}
+	secondBody := framework.ReadAllAndClose(t, secondResp)
+	if secondResp.StatusCode != http.StatusOK {
+		t.Fatalf("second GET expected 200 via replica proxy fallback, got %d", secondResp.StatusCode)
+	}
+	if len(secondBody) != len(largePayload) {
+		t.Fatalf("second GET proxied body size mismatch: got %d want %d", len(secondBody), len(largePayload))
+	}
+}
+
+func TestDownloadLimitProxiedRequestSkipsReplicaFallbackAndTimesOut(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P8()
+	profile.ReadMode = "proxy"
+	clusterHarness := framework.StartDualVolumeCluster(t, profile)
+
+	conn0, grpc0 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, grpc1 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	const volumeID = uint32(106)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	req := &volume_server_pb.AllocateVolumeRequest{
+		VolumeId:    volumeID,
+		Replication: "001",
+		Version:     uint32(needle.GetCurrentVersion()),
+	}
+	if _, err := grpc0.AllocateVolume(ctx, req); err != nil {
+		t.Fatalf("allocate replicated volume on node0: %v", err)
+	}
+	if _, err := grpc1.AllocateVolume(ctx, req); err != nil {
+		t.Fatalf("allocate replicated volume on node1: %v", err)
+	}
+
+	largePayload := make([]byte, 12*1024*1024)
+	for i := range largePayload {
+		largePayload[i] = byte(i % 251)
+	}
+	fid := framework.NewFileID(volumeID, 880202, 0x0A0B0D0E)
+	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(0), fid, largePayload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("replicated large upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	// Ensure replica path is actually available, so a non-proxied request would proxy.
+	replicaReadURL := clusterHarness.VolumeAdminURL(1) + "/" + fid
+	if !waitForHTTPStatus(t, framework.NewHTTPClient(), replicaReadURL, http.StatusOK, 10*time.Second, func(resp *http.Response) {
+		_ = framework.ReadAllAndClose(t, resp)
+	}) {
+		t.Fatalf("replica did not become readable within deadline")
+	}
+
+	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid))
+	if err != nil {
+		t.Fatalf("first GET failed: %v", err)
+	}
+	if firstResp.StatusCode != http.StatusOK {
+		_ = framework.ReadAllAndClose(t, firstResp)
+		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
+	}
+	defer firstResp.Body.Close()
+
+	time.Sleep(300 * time.Millisecond)
+
+	// proxied=true should bypass replica fallback and hit wait/timeout branch.
+	secondResp, err := framework.NewHTTPClient().Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL(0)+"/"+fid+"?proxied=true"))
+	if err != nil {
+		t.Fatalf("second GET failed: %v", err)
+	}
+	_ = framework.ReadAllAndClose(t, secondResp)
+	if secondResp.StatusCode != http.StatusTooManyRequests {
+		t.Fatalf("second GET with proxied=true expected 429 timeout path, got %d", secondResp.StatusCode)
+	}
+}
+
+func TestUploadLimitDisabledAllowsConcurrentUploads(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(107)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	const blockedUploadSize = 2 * 1024 * 1024
+	unblockFirstUpload := make(chan struct{})
+	firstUploadDone := make(chan error, 1)
+	firstFID := framework.NewFileID(volumeID, 880301, 0x1A2B3C5D)
+	go func() {
+		req, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+firstFID, &pausableReader{
+			remaining:  blockedUploadSize,
+			pauseAfter: 1,
+			unblock:    unblockFirstUpload,
+		})
+		if err != nil {
+			firstUploadDone <- err
+			return
+		}
+		req.Header.Set("Content-Type", "application/octet-stream")
+		req.ContentLength = blockedUploadSize
+
+		resp, err := (&http.Client{}).Do(req)
+		if resp != nil {
+			_ = framework.ReadAllAndClose(t, resp)
+		}
+		firstUploadDone <- err
+	}()
+
+	time.Sleep(300 * time.Millisecond)
+
+	secondFID := framework.NewFileID(volumeID, 880302, 0x1A2B3C5E)
+	secondResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), secondFID, []byte("no-limit-second-upload"))
+	_ = framework.ReadAllAndClose(t, secondResp)
+	if secondResp.StatusCode != http.StatusCreated {
+		t.Fatalf("second upload with disabled limit expected 201, got %d", secondResp.StatusCode)
+	}
+
+	close(unblockFirstUpload)
+	select {
+	case <-firstUploadDone:
+	case <-time.After(5 * time.Second):
+		t.Fatalf("timed out waiting for first upload completion")
+	}
+}
+
+func TestDownloadLimitDisabledAllowsConcurrentDownloads(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(108)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	largePayload := make([]byte, 12*1024*1024)
+	for i := range largePayload {
+		largePayload[i] = byte(i % 251)
+	}
+	fid := framework.NewFileID(volumeID, 880401, 0x20304050)
+	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid))
+	if err != nil {
+		t.Fatalf("first GET failed: %v", err)
+	}
+	if firstResp.StatusCode != http.StatusOK {
+		_ = framework.ReadAllAndClose(t, firstResp)
+		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
+	}
+	defer firstResp.Body.Close()
+
+	time.Sleep(300 * time.Millisecond)
+
+	secondResp := framework.ReadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid)
+	secondBody := framework.ReadAllAndClose(t, secondResp)
+	if secondResp.StatusCode != http.StatusOK {
+		t.Fatalf("second GET with disabled limit expected 200, got %d", secondResp.StatusCode)
+	}
+	if len(secondBody) != len(largePayload) {
+		t.Fatalf("second GET body size mismatch: got %d want %d", len(secondBody), len(largePayload))
+	}
+}
+
+func TestDownloadLimitInvalidVidWhileOverLimitReturnsBadRequest(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P8())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(110)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	largePayload := make([]byte, 12*1024*1024)
+	for i := range largePayload {
+		largePayload[i] = byte(i % 251)
+	}
+	fid := framework.NewFileID(volumeID, 880501, 0x50607080)
+	uploadResp := framework.UploadBytes(t, framework.NewHTTPClient(), clusterHarness.VolumeAdminURL(), fid, largePayload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	firstResp, err := (&http.Client{}).Do(mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid))
+	if err != nil {
+		t.Fatalf("first GET failed: %v", err)
+	}
+	if firstResp.StatusCode != http.StatusOK {
+		_ = framework.ReadAllAndClose(t, firstResp)
+		t.Fatalf("first GET expected 200, got %d", firstResp.StatusCode)
+	}
+	defer firstResp.Body.Close()
+
+	time.Sleep(300 * time.Millisecond)
+
+	invalidReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/not-a-vid,1234567890ab")
+	invalidResp := framework.DoRequest(t, framework.NewHTTPClient(), invalidReq)
+	_ = framework.ReadAllAndClose(t, invalidResp)
+	if invalidResp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("invalid vid while over limit expected 400, got %d", invalidResp.StatusCode)
+	}
+}

test/volume_server/http/write_delete_variants_test.go

@@ -0,0 +1,118 @@
+package volume_server_http_test
+
+import (
+	"encoding/json"
+	"net/http"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+func TestWriteUnchangedAndDeleteEdgeVariants(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(87)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	const key = uint64(999001)
+	const cookie = uint32(0xDEADBEEF)
+	fid := framework.NewFileID(volumeID, key, cookie)
+	client := framework.NewHTTPClient()
+	payload := []byte("unchanged-write-content")
+
+	firstUpload := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
+	firstUploadResp := framework.DoRequest(t, client, firstUpload)
+	_ = framework.ReadAllAndClose(t, firstUploadResp)
+	if firstUploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("first upload expected 201, got %d", firstUploadResp.StatusCode)
+	}
+
+	secondUpload := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, payload)
+	secondUploadResp := framework.DoRequest(t, client, secondUpload)
+	_ = framework.ReadAllAndClose(t, secondUploadResp)
+	if secondUploadResp.StatusCode != http.StatusNoContent {
+		t.Fatalf("second unchanged upload expected 204, got %d", secondUploadResp.StatusCode)
+	}
+	if secondUploadResp.Header.Get("ETag") == "" {
+		t.Fatalf("second unchanged upload expected ETag header")
+	}
+
+	wrongCookieFid := framework.NewFileID(volumeID, key, cookie+1)
+	wrongCookieDelete := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+wrongCookieFid))
+	_ = framework.ReadAllAndClose(t, wrongCookieDelete)
+	if wrongCookieDelete.StatusCode != http.StatusBadRequest {
+		t.Fatalf("delete with mismatched cookie expected 400, got %d", wrongCookieDelete.StatusCode)
+	}
+
+	missingDelete := framework.DoRequest(t, client, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+framework.NewFileID(volumeID, key+1, cookie)))
+	missingDeleteBody := framework.ReadAllAndClose(t, missingDelete)
+	if missingDelete.StatusCode != http.StatusNotFound {
+		t.Fatalf("delete missing needle expected 404, got %d", missingDelete.StatusCode)
+	}
+
+	var payloadMap map[string]int64
+	if err := json.Unmarshal(missingDeleteBody, &payloadMap); err != nil {
+		t.Fatalf("decode delete missing response: %v", err)
+	}
+	if payloadMap["size"] != 0 {
+		t.Fatalf("delete missing needle expected size=0, got %d", payloadMap["size"])
+	}
+}
+
+func TestDeleteTimestampOverrideKeepsReadDeletedLastModifiedParity(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(88)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 999002, 0xABCD1234)
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("delete-ts-override"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	beforeDeleteResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
+	_ = framework.ReadAllAndClose(t, beforeDeleteResp)
+	if beforeDeleteResp.StatusCode != http.StatusOK {
+		t.Fatalf("pre-delete read expected 200, got %d", beforeDeleteResp.StatusCode)
+	}
+	lastModifiedBeforeDelete := beforeDeleteResp.Header.Get("Last-Modified")
+	if lastModifiedBeforeDelete == "" {
+		t.Fatalf("expected Last-Modified before delete")
+	}
+
+	deleteReq := mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid+"?ts=1700000000")
+	deleteResp := framework.DoRequest(t, client, deleteReq)
+	_ = framework.ReadAllAndClose(t, deleteResp)
+	if deleteResp.StatusCode != http.StatusAccepted {
+		t.Fatalf("delete with ts override expected 202, got %d", deleteResp.StatusCode)
+	}
+
+	readDeletedResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true"))
+	_ = framework.ReadAllAndClose(t, readDeletedResp)
+	if readDeletedResp.StatusCode != http.StatusOK {
+		t.Fatalf("readDeleted after ts override expected 200, got %d", readDeletedResp.StatusCode)
+	}
+	lastModified := readDeletedResp.Header.Get("Last-Modified")
+	if lastModified == "" {
+		t.Fatalf("expected Last-Modified header on readDeleted response")
+	}
+	if lastModified != lastModifiedBeforeDelete {
+		t.Fatalf("expected readDeleted Last-Modified parity with pre-delete header, got %q want %q", lastModified, lastModifiedBeforeDelete)
+	}
+}

test/volume_server/http/write_error_variants_test.go

@@ -0,0 +1,74 @@
+package volume_server_http_test
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+func TestWriteInvalidVidAndFidReturnBadRequest(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	client := framework.NewHTTPClient()
+
+	invalidVidReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/invalid,12345678", []byte("x"))
+	invalidVidResp := framework.DoRequest(t, client, invalidVidReq)
+	_ = framework.ReadAllAndClose(t, invalidVidResp)
+	if invalidVidResp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("write with invalid vid expected 400, got %d", invalidVidResp.StatusCode)
+	}
+
+	invalidFidReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/1,bad", []byte("x"))
+	invalidFidResp := framework.DoRequest(t, client, invalidFidReq)
+	_ = framework.ReadAllAndClose(t, invalidFidResp)
+	if invalidFidResp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("write with invalid fid expected 400, got %d", invalidFidResp.StatusCode)
+	}
+}
+
+func TestWriteMalformedMultipartAndMD5Mismatch(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(98)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 772001, 0x1A2B3C4D)
+
+	malformedMultipartReq, err := http.NewRequest(http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+fid, strings.NewReader("not-a-valid-multipart-body"))
+	if err != nil {
+		t.Fatalf("create malformed multipart request: %v", err)
+	}
+	malformedMultipartReq.Header.Set("Content-Type", "multipart/form-data")
+	malformedMultipartResp := framework.DoRequest(t, client, malformedMultipartReq)
+	malformedMultipartBody := framework.ReadAllAndClose(t, malformedMultipartResp)
+	if malformedMultipartResp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("malformed multipart write expected 400, got %d", malformedMultipartResp.StatusCode)
+	}
+	if !strings.Contains(strings.ToLower(string(malformedMultipartBody)), "boundary") {
+		t.Fatalf("malformed multipart response should mention boundary parse failure, got %q", string(malformedMultipartBody))
+	}
+
+	md5MismatchReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, []byte("content-md5-mismatch-body"))
+	md5MismatchReq.Header.Set("Content-MD5", "AAAAAAAAAAAAAAAAAAAAAA==")
+	md5MismatchResp := framework.DoRequest(t, client, md5MismatchReq)
+	md5MismatchBody := framework.ReadAllAndClose(t, md5MismatchResp)
+	if md5MismatchResp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("content-md5 mismatch write expected 400, got %d", md5MismatchResp.StatusCode)
+	}
+	if !strings.Contains(string(md5MismatchBody), "Content-MD5") {
+		t.Fatalf("content-md5 mismatch response should mention Content-MD5, got %q", string(md5MismatchBody))
+	}
+}

test/volume_server/matrix/config_profiles.go

@@ -0,0 +1,63 @@
+package matrix
+
+import "time"
+
+// Profile describes one runtime test matrix configuration.
+type Profile struct {
+	Name string
+
+	ReadMode        string
+	SplitPublicPort bool
+
+	EnableJWT      bool
+	JWTSigningKey  string
+	JWTReadKey     string
+	EnableMaintain bool
+
+	ConcurrentUploadLimitMB   int
+	ConcurrentDownloadLimitMB int
+	InflightUploadTimeout     time.Duration
+	InflightDownloadTimeout   time.Duration
+
+	ReplicatedLayout bool
+	HasErasureCoding bool
+	HasRemoteTier    bool
+}
+
+// P1 is the baseline profile: one volume server, no JWT, proxy read mode.
+func P1() Profile {
+	return Profile{
+		Name:            "P1",
+		ReadMode:        "proxy",
+		SplitPublicPort: false,
+	}
+}
+
+// P2 uses split public/admin ports to verify public read-only behavior.
+func P2() Profile {
+	p := P1()
+	p.Name = "P2"
+	p.SplitPublicPort = true
+	return p
+}
+
+// P3 enables JWT verification for read/write flows.
+func P3() Profile {
+	p := P1()
+	p.Name = "P3"
+	p.EnableJWT = true
+	p.JWTSigningKey = "volume-server-write-key"
+	p.JWTReadKey = "volume-server-read-key"
+	return p
+}
+
+// P8 enables upload/download throttling branches.
+func P8() Profile {
+	p := P1()
+	p.Name = "P8"
+	p.ConcurrentUploadLimitMB = 1
+	p.ConcurrentDownloadLimitMB = 1
+	p.InflightUploadTimeout = 2 * time.Second
+	p.InflightDownloadTimeout = 2 * time.Second
+	return p
+}

weed/admin/dash/admin_data.go

@@ -9,13 +9,14 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/iam"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 )
 
 // Access key status constants
 const (
-	AccessKeyStatusActive   = "Active"
-	AccessKeyStatusInactive = "Inactive"
+	AccessKeyStatusActive   = iam.AccessKeyStatusActive
+	AccessKeyStatusInactive = iam.AccessKeyStatusInactive
 )
 
 type AdminData struct {

weed/admin/dash/admin_server.go

@@ -4,12 +4,15 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
+	"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/credential"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -115,6 +118,9 @@ type AdminServer struct {
 
 	s3TablesManager *s3tables.Manager
 	icebergPort     int
+
+	// Plugin system manager
+	pluginManager interface{}
 }
 
 // Type definitions moved to types.go
@@ -226,6 +232,9 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
 		}()
 	}
 
+	// Initialize plugin manager
+	server.initPluginManager(dataDir)
+
 	return server
 }
 
@@ -246,6 +255,38 @@ func (s *AdminServer) GetCredentialManager() *credential.CredentialManager {
 	return s.credentialManager
 }
 
+// initPluginManager initializes the plugin manager
+func (s *AdminServer) initPluginManager(dataDir string) {
+	// Create plugin configuration directory if it doesn't exist
+	pluginConfigDir := filepath.Join(dataDir, "plugins")
+	if err := os.MkdirAll(pluginConfigDir, 0755); err != nil {
+		glog.Warningf("Failed to create plugin config directory: %v", err)
+		return
+	}
+
+	// Create plugin manager with default configuration
+	config := plugin.DefaultManagerConfig(pluginConfigDir)
+	pm, err := plugin.NewManager(config)
+	if err != nil {
+		glog.Warningf("Failed to initialize plugin manager: %v", err)
+		return
+	}
+
+	// Store the plugin manager
+	s.pluginManager = pm
+	glog.Infof("Plugin manager initialized successfully")
+}
+
+// GetPluginManager returns the plugin manager
+func (s *AdminServer) GetPluginManager() interface{} {
+	return s.pluginManager
+}
+
+// SetPluginManager sets the plugin manager
+func (s *AdminServer) SetPluginManager(pm interface{}) {
+	s.pluginManager = pm
+}
+
 // Filer discovery methods moved to client_management.go
 
 // Client management methods moved to client_management.go

weed/admin/dash/s3tables_management.go

@@ -579,9 +579,6 @@ func (s *AdminServer) ListS3TablesBucketsAPI(c *gin.Context) {
 }
 
 func (s *AdminServer) CreateS3TablesBucket(c *gin.Context) {
-	if !requireSessionCSRFToken(c) {
-		return
-	}
 	var req struct {
 		Name  string            `json:"name"`
 		Tags  map[string]string `json:"tags"`
@@ -667,9 +664,6 @@ func (s *AdminServer) SetTableBucketOwner(ctx context.Context, bucketName, owner
 }
 
 func (s *AdminServer) DeleteS3TablesBucket(c *gin.Context) {
-	if !requireSessionCSRFToken(c) {
-		return
-	}
 	bucketArn := c.Query("bucket")
 	if bucketArn == "" {
 		c.JSON(400, gin.H{"error": "Bucket ARN is required"})

weed/admin/dash/service_account_management.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/iam"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 )
 
@@ -21,8 +22,8 @@ const (
 	accessKeyPrefix      = "ABIA" // Service account access keys use ABIA prefix
 
 	// Status constants
-	StatusActive   = "Active"
-	StatusInactive = "Inactive"
+	StatusActive   = iam.AccessKeyStatusActive
+	StatusInactive = iam.AccessKeyStatusInactive
 )
 
 // GetServiceAccounts returns all service accounts, optionally filtered by parent user

weed/admin/dash/worker_grpc_server.go

@@ -9,8 +9,10 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
+	"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -94,6 +96,19 @@ func (s *WorkerGrpcServer) StartWithTLS(port int) error {
 
 	worker_pb.RegisterWorkerServiceServer(grpcServer, s)
 
+	// Register plugin service if plugin manager is available
+	if s.adminServer.GetPluginManager() != nil {
+		// Cast the interface{} to *plugin.Manager
+		if pm, ok := s.adminServer.GetPluginManager().(*plugin.Manager); ok {
+			if pluginGrpcServer := pm.GetGRPCServer(); pluginGrpcServer != nil {
+				plugin_pb.RegisterPluginServiceServer(grpcServer, pluginGrpcServer)
+				plugin_pb.RegisterAdminQueryServiceServer(grpcServer, pluginGrpcServer)
+				plugin_pb.RegisterAdminCommandServiceServer(grpcServer, pluginGrpcServer)
+				glog.Infof("Registered plugin services on worker gRPC server")
+			}
+		}
+	}
+
 	s.grpcServer = grpcServer
 	s.listener = listener
 	s.running = true

weed/admin/handlers/admin_handlers.go

@@ -3,11 +3,13 @@ package handlers
 import (
 	"net/http"
 	"net/url"
+	"sort"
 	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
+	"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
 	"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
 	"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
@@ -26,6 +28,7 @@ type AdminHandlers struct {
 	maintenanceHandlers    *MaintenanceHandlers
 	mqHandlers             *MessageQueueHandlers
 	serviceAccountHandlers *ServiceAccountHandlers
+	pluginHandlers         *PluginHandlers
 }
 
 // NewAdminHandlers creates a new instance of AdminHandlers
@@ -38,6 +41,14 @@ func NewAdminHandlers(adminServer *dash.AdminServer) *AdminHandlers {
 	maintenanceHandlers := NewMaintenanceHandlers(adminServer)
 	mqHandlers := NewMessageQueueHandlers(adminServer)
 	serviceAccountHandlers := NewServiceAccountHandlers(adminServer)
+
+	// Get plugin manager from admin server (may be nil)
+	var pluginMgr interface{}
+	if pm := adminServer.GetPluginManager(); pm != nil {
+		pluginMgr = pm
+	}
+	pluginHandlers := NewPluginHandlers(adminServer, pluginMgr)
+
 	return &AdminHandlers{
 		adminServer:            adminServer,
 		authHandlers:           authHandlers,
@@ -48,6 +59,7 @@ func NewAdminHandlers(adminServer *dash.AdminServer) *AdminHandlers {
 		maintenanceHandlers:    maintenanceHandlers,
 		mqHandlers:             mqHandlers,
 		serviceAccountHandlers: serviceAccountHandlers,
+		pluginHandlers:         pluginHandlers,
 	}
 }
 
@@ -119,6 +131,11 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
 		protected.GET("/mq/topics", h.mqHandlers.ShowTopics)
 		protected.GET("/mq/topics/:namespace/:topic", h.mqHandlers.ShowTopicDetails)
 
+		// Plugin management routes
+		protected.GET("/plugins", h.ShowPlugins)
+		protected.GET("/plugins/jobs/:jobType", h.ShowPluginJobs)
+		protected.GET("/plugins/config/:jobType", h.ShowPluginConfig)
+
 		// Maintenance system routes
 		protected.GET("/maintenance", h.maintenanceHandlers.ShowMaintenanceQueue)
 		protected.GET("/maintenance/workers", h.maintenanceHandlers.ShowMaintenanceWorkers)
@@ -250,6 +267,19 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
 				mqApi.POST("/topics/retention/update", dash.RequireWriteAccess(), h.mqHandlers.UpdateTopicRetentionAPI)
 				mqApi.POST("/retention/purge", dash.RequireWriteAccess(), h.adminServer.TriggerTopicRetentionPurgeAPI)
 			}
+
+			// Plugin API routes
+			pluginApi := api.Group("/plugin")
+			{
+				pluginApi.GET("/list", h.pluginHandlers.ListPluginsAPI)
+				pluginApi.GET("/jobs/by-type/:type", h.pluginHandlers.ListJobsAPI)
+				pluginApi.GET("/config/:type", h.pluginHandlers.GetConfigAPI)
+				pluginApi.POST("/config/:type/apply", dash.RequireWriteAccess(), h.pluginHandlers.SaveConfigAPI)
+				pluginApi.GET("/detection/history/:type", h.pluginHandlers.GetDetectionHistoryAPI)
+				pluginApi.GET("/execution/history/:type", h.pluginHandlers.GetExecutionHistoryAPI)
+				pluginApi.POST("/trigger-detection/:type", dash.RequireWriteAccess(), h.pluginHandlers.TriggerDetectionAPI)
+				pluginApi.POST("/cancel-job/:id", dash.RequireWriteAccess(), h.pluginHandlers.CancelJobAPI)
+			}
 		}
 	} else {
 		// No authentication required - all routes are public
@@ -292,6 +322,11 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
 		r.GET("/mq/topics", h.mqHandlers.ShowTopics)
 		r.GET("/mq/topics/:namespace/:topic", h.mqHandlers.ShowTopicDetails)
 
+		// Plugin management routes
+		r.GET("/plugins", h.ShowPlugins)
+		r.GET("/plugins/jobs/:jobType", h.ShowPluginJobs)
+		r.GET("/plugins/config/:jobType", h.ShowPluginConfig)
+
 		// Maintenance system routes
 		r.GET("/maintenance", h.maintenanceHandlers.ShowMaintenanceQueue)
 		r.GET("/maintenance/workers", h.maintenanceHandlers.ShowMaintenanceWorkers)
@@ -422,6 +457,19 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
 				mqApi.POST("/topics/retention/update", h.mqHandlers.UpdateTopicRetentionAPI)
 				mqApi.POST("/retention/purge", h.adminServer.TriggerTopicRetentionPurgeAPI)
 			}
+
+		// Plugin API routes
+		pluginApi := api.Group("/plugin")
+		{
+			pluginApi.GET("/list", h.pluginHandlers.ListPluginsAPI)
+			pluginApi.GET("/jobs/by-type/:type", h.pluginHandlers.ListJobsAPI)
+			pluginApi.GET("/config/:type", h.pluginHandlers.GetConfigAPI)
+			pluginApi.POST("/config/:type/apply", h.pluginHandlers.SaveConfigAPI)
+			pluginApi.GET("/detection/history/:type", h.pluginHandlers.GetDetectionHistoryAPI)
+			pluginApi.GET("/execution/history/:type", h.pluginHandlers.GetExecutionHistoryAPI)
+			pluginApi.POST("/trigger-detection/:type", h.pluginHandlers.TriggerDetectionAPI)
+			pluginApi.POST("/cancel-job/:id", h.pluginHandlers.CancelJobAPI)
+		}
 		}
 	}
 }
@@ -670,4 +718,97 @@ func (h *AdminHandlers) getAdminData(c *gin.Context) dash.AdminData {
 	return adminData
 }
 
+// ShowPlugins displays the plugins overview page
+func (h *AdminHandlers) ShowPlugins(c *gin.Context) {
+	plugins := []map[string]interface{}{}
+	jobTypes := make(map[string]interface{})
+
+	// Get plugin manager from server
+	if pm := h.adminServer.GetPluginManager(); pm != nil {
+		// Cast to *plugin.Manager
+		if pluginMgr, ok := pm.(*plugin.Manager); ok {
+			// Get list of connected plugins
+			connectedPlugins := pluginMgr.ListPlugins(false)
+			for _, p := range connectedPlugins {
+				plugins = append(plugins, map[string]interface{}{
+					"id":            p.ID,
+					"name":          p.Name,
+					"version":       p.Version,
+					"status":        p.Status,
+					"capabilities":  p.Capabilities,
+					"activeJobs":    p.ActiveJobs,
+					"completedJobs": p.CompletedJobs,
+					"failedJobs":    p.FailedJobs,
+					"connectedAt":   p.ConnectedAt,
+					"lastHeartbeat": p.LastHeartbeat,
+				})
+				
+				// Build job types map
+				for _, cap := range p.Capabilities {
+					if _, exists := jobTypes[cap]; !exists {
+						jobTypes[cap] = map[string]interface{}{
+							"type":        cap,
+							"description": cap,
+							"pluginCount": 0,
+						}
+					}
+					// Increment plugin count for this capability
+					if capData, ok := jobTypes[cap].(map[string]interface{}); ok {
+						capData["pluginCount"] = capData["pluginCount"].(int) + 1
+					}
+				}
+			}
+		}
+	}
+
+	// Sort plugins by ID
+	sort.Slice(plugins, func(i, j int) bool {
+		return plugins[i]["id"].(string) < plugins[j]["id"].(string)
+	})
+
+	component := app.PluginsOverview(app.PluginsPageData{
+		Plugins:  plugins,
+		JobTypes: jobTypes,
+	})
+
+	htmlContent := layout.Layout(c, component)
+	htmlContent.Render(c.Request.Context(), c.Writer)
+}
+
+// ShowPluginJobs displays the job monitoring page for a specific type
+func (h *AdminHandlers) ShowPluginJobs(c *gin.Context) {
+	jobType := c.Param("jobType")
+	jobs := []interface{}{}
+	stateFilter := c.Query("state")
+
+	component := app.PluginJobsMonitoring(app.PluginJobsPageData{
+		JobType:     jobType,
+		Jobs:        jobs,
+		StateFilter: stateFilter,
+	})
+
+	htmlContent := layout.Layout(c, component)
+	htmlContent.Render(c.Request.Context(), c.Writer)
+}
+
+// ShowPluginConfig displays the configuration page for a job type
+func (h *AdminHandlers) ShowPluginConfig(c *gin.Context) {
+	jobType := c.Param("jobType")
+	activeTab := c.Query("tab")
+	if activeTab == "" {
+		activeTab = "config"
+	}
+
+	component := app.PluginConfiguration(app.PluginConfigPageData{
+		JobType:          jobType,
+		Config:           app.JobTypeConfig{},
+		DetectionHistory: []interface{}{},
+		ExecutionHistory: []interface{}{},
+		ActiveTab:        activeTab,
+	})
+
+	htmlContent := layout.Layout(c, component)
+	htmlContent.Render(c.Request.Context(), c.Writer)
+}
+
 // Helper functions

weed/admin/handlers/plugin_handlers.go

@@ -0,0 +1,95 @@
+package handlers
+
+import (
+"net/http"
+
+"github.com/gin-gonic/gin"
+)
+
+type PluginHandlers struct {
+adminServer interface{}
+pluginMgr   interface{}
+}
+
+func NewPluginHandlers(adminServer interface{}, pluginMgr interface{}) *PluginHandlers {
+return &PluginHandlers{
+adminServer: adminServer,
+pluginMgr:   pluginMgr,
+}
+}
+
+// ListPluginsAPI returns list of connected plugins
+func (h *PluginHandlers) ListPluginsAPI(c *gin.Context) {
+result := []map[string]interface{}{}
+c.JSON(http.StatusOK, result)
+}
+
+// ListJobsAPI returns jobs for a specific type
+func (h *PluginHandlers) ListJobsAPI(c *gin.Context) {
+jobType := c.Param("type")
+result := map[string]interface{}{
+"job_type": jobType,
+"jobs":     []interface{}{},
+}
+c.JSON(http.StatusOK, result)
+}
+
+// GetConfigAPI returns configuration for a job type
+func (h *PluginHandlers) GetConfigAPI(c *gin.Context) {
+jobType := c.Param("type")
+result := map[string]interface{}{
+"type": jobType,
+}
+c.JSON(http.StatusOK, result)
+}
+
+// SaveConfigAPI saves configuration for a job type
+func (h *PluginHandlers) SaveConfigAPI(c *gin.Context) {
+jobType := c.Param("type")
+result := map[string]string{
+"status": "saved",
+"type":   jobType,
+}
+c.JSON(http.StatusOK, result)
+}
+
+// GetDetectionHistoryAPI returns detection history for a job type
+func (h *PluginHandlers) GetDetectionHistoryAPI(c *gin.Context) {
+jobType := c.Param("type")
+result := map[string]interface{}{
+"job_type": jobType,
+"records":  []interface{}{},
+}
+c.JSON(http.StatusOK, result)
+}
+
+// GetExecutionHistoryAPI returns execution history for a job type
+func (h *PluginHandlers) GetExecutionHistoryAPI(c *gin.Context) {
+jobType := c.Param("type")
+result := map[string]interface{}{
+"job_type": jobType,
+"records":  []interface{}{},
+}
+c.JSON(http.StatusOK, result)
+}
+
+// TriggerDetectionAPI manually triggers detection
+func (h *PluginHandlers) TriggerDetectionAPI(c *gin.Context) {
+jobType := c.Param("type")
+result := map[string]interface{}{
+"status":   "triggered",
+"job_type": jobType,
+"job_ids":  []string{},
+}
+c.JSON(http.StatusOK, result)
+}
+
+// CancelJobAPI cancels a job
+func (h *PluginHandlers) CancelJobAPI(c *gin.Context) {
+jobID := c.Param("id")
+result := map[string]string{
+"status": "cancelled",
+"job_id": jobID,
+}
+c.JSON(http.StatusOK, result)
+}

weed/admin/plugin/config_manager.go

@@ -0,0 +1,376 @@
+package plugin
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+// ConfigManager handles JSON-based configuration persistence
+type ConfigManager struct {
+	mu              sync.RWMutex
+	configDir       string
+	defaultConfigFile string
+	pluginConfigs   map[string]*PluginConfig
+	configVersions  map[string]int64
+	lastModified    map[string]time.Time
+	backupDir       string
+	maxBackups      int
+}
+
+// NewConfigManager creates a new configuration manager
+func NewConfigManager(configDir string) (*ConfigManager, error) {
+	// Ensure config directory exists
+	if err := os.MkdirAll(configDir, 0755); err != nil {
+		return nil, fmt.Errorf("failed to create config directory: %w", err)
+	}
+
+	backupDir := filepath.Join(configDir, "backups")
+	if err := os.MkdirAll(backupDir, 0755); err != nil {
+		return nil, fmt.Errorf("failed to create backup directory: %w", err)
+	}
+
+	return &ConfigManager{
+		configDir:         configDir,
+		defaultConfigFile: filepath.Join(configDir, "plugins.json"),
+		pluginConfigs:     make(map[string]*PluginConfig),
+		configVersions:    make(map[string]int64),
+		lastModified:      make(map[string]time.Time),
+		backupDir:         backupDir,
+		maxBackups:        10,
+	}, nil
+}
+
+// SaveConfig persists a plugin configuration to disk
+func (cm *ConfigManager) SaveConfig(config *PluginConfig, backup bool) error {
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+
+	if backup {
+		if err := cm.backupExistingConfig(config.PluginID); err != nil {
+			return fmt.Errorf("failed to backup config: %w", err)
+		}
+	}
+
+	configFile := filepath.Join(cm.configDir, fmt.Sprintf("%s.json", config.PluginID))
+	configData := map[string]interface{}{
+		"plugin_id":              config.PluginID,
+		"properties":             config.Properties,
+		"job_types":              config.JobTypes,
+		"max_retries":            config.MaxRetries,
+		"health_check_interval":  config.HealthCheckInterval.String(),
+		"job_timeout":            config.JobTimeout.String(),
+		"environment":            config.Environment,
+	}
+
+	data, err := json.MarshalIndent(configData, "", "  ")
+	if err != nil {
+		return fmt.Errorf("failed to marshal config: %w", err)
+	}
+
+	if err := ioutil.WriteFile(configFile, data, 0644); err != nil {
+		return fmt.Errorf("failed to write config file: %w", err)
+	}
+
+	// Update in-memory state
+	cm.pluginConfigs[config.PluginID] = config
+	cm.configVersions[config.PluginID]++
+	cm.lastModified[config.PluginID] = time.Now()
+
+	return nil
+}
+
+// LoadConfig loads a plugin configuration from disk
+func (cm *ConfigManager) LoadConfig(pluginID string) (*PluginConfig, error) {
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+
+	configFile := filepath.Join(cm.configDir, fmt.Sprintf("%s.json", pluginID))
+
+	data, err := ioutil.ReadFile(configFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read config file: %w", err)
+	}
+
+	var configData map[string]interface{}
+	if err := json.Unmarshal(data, &configData); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal config: %w", err)
+	}
+
+	config := &PluginConfig{
+		PluginID:    pluginID,
+		Properties:  make(map[string]string),
+		JobTypes:    make(map[string]*JobTypeConfig),
+		Environment: make(map[string]string),
+	}
+
+	// Parse basic fields
+	if props, ok := configData["properties"].(map[string]interface{}); ok {
+		for k, v := range props {
+			if str, ok := v.(string); ok {
+				config.Properties[k] = str
+			}
+		}
+	}
+
+	if maxRetries, ok := configData["max_retries"].(float64); ok {
+		config.MaxRetries = int(maxRetries)
+	}
+
+	if hcInterval, ok := configData["health_check_interval"].(string); ok {
+		if duration, err := time.ParseDuration(hcInterval); err == nil {
+			config.HealthCheckInterval = duration
+		}
+	}
+
+	if timeout, ok := configData["job_timeout"].(string); ok {
+		if duration, err := time.ParseDuration(timeout); err == nil {
+			config.JobTimeout = duration
+		}
+	}
+
+	if env, ok := configData["environment"].(map[string]interface{}); ok {
+		for k, v := range env {
+			if str, ok := v.(string); ok {
+				config.Environment[k] = str
+			}
+		}
+	}
+
+	// Parse job types
+	if jobTypes, ok := configData["job_types"].(map[string]interface{}); ok {
+		for jobType, typeConfig := range jobTypes {
+			if typeCfg, ok := typeConfig.(map[string]interface{}); ok {
+				jtc := &JobTypeConfig{
+					Type:       jobType,
+					Parameters: make(map[string]string),
+				}
+
+				if enabled, ok := typeCfg["enabled"].(bool); ok {
+					jtc.Enabled = enabled
+				}
+
+				if priority, ok := typeCfg["priority"].(float64); ok {
+					jtc.Priority = int(priority)
+				}
+
+				if interval, ok := typeCfg["interval"].(string); ok {
+					if duration, err := time.ParseDuration(interval); err == nil {
+						jtc.Interval = duration
+					}
+				}
+
+				if maxConcurrent, ok := typeCfg["max_concurrent"].(float64); ok {
+					jtc.MaxConcurrent = int(maxConcurrent)
+				}
+
+				if params, ok := typeCfg["parameters"].(map[string]interface{}); ok {
+					for pk, pv := range params {
+						if str, ok := pv.(string); ok {
+							jtc.Parameters[pk] = str
+						}
+					}
+				}
+
+				config.JobTypes[jobType] = jtc
+			}
+		}
+	}
+
+	cm.pluginConfigs[pluginID] = config
+	cm.configVersions[pluginID]++
+	cm.lastModified[pluginID] = time.Now()
+
+	return config, nil
+}
+
+// GetConfig retrieves a configuration from memory
+func (cm *ConfigManager) GetConfig(pluginID string) (*PluginConfig, bool) {
+	cm.mu.RLock()
+	defer cm.mu.RUnlock()
+
+	config, exists := cm.pluginConfigs[pluginID]
+	return config, exists
+}
+
+// ListConfigs returns all loaded configurations
+func (cm *ConfigManager) ListConfigs() map[string]*PluginConfig {
+	cm.mu.RLock()
+	defer cm.mu.RUnlock()
+
+	result := make(map[string]*PluginConfig)
+	for pluginID, config := range cm.pluginConfigs {
+		result[pluginID] = config
+	}
+	return result
+}
+
+// DeleteConfig removes a configuration
+func (cm *ConfigManager) DeleteConfig(pluginID string) error {
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+
+	configFile := filepath.Join(cm.configDir, fmt.Sprintf("%s.json", pluginID))
+	if err := os.Remove(configFile); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to delete config file: %w", err)
+	}
+
+	delete(cm.pluginConfigs, pluginID)
+	delete(cm.configVersions, pluginID)
+	delete(cm.lastModified, pluginID)
+
+	return nil
+}
+
+// GetVersion returns the version number of a configuration
+func (cm *ConfigManager) GetVersion(pluginID string) int64 {
+	cm.mu.RLock()
+	defer cm.mu.RUnlock()
+
+	return cm.configVersions[pluginID]
+}
+
+// GetLastModified returns the last modification time of a configuration
+func (cm *ConfigManager) GetLastModified(pluginID string) time.Time {
+	cm.mu.RLock()
+	defer cm.mu.RUnlock()
+
+	return cm.lastModified[pluginID]
+}
+
+// backupExistingConfig creates a backup of an existing configuration
+func (cm *ConfigManager) backupExistingConfig(pluginID string) error {
+	configFile := filepath.Join(cm.configDir, fmt.Sprintf("%s.json", pluginID))
+
+	// Check if file exists
+	if _, err := os.Stat(configFile); os.IsNotExist(err) {
+		return nil // No existing config to back up
+	}
+
+	// Create backup filename with timestamp
+	backupFilename := fmt.Sprintf("%s_%d.json.bak", pluginID, time.Now().Unix())
+	backupFile := filepath.Join(cm.backupDir, backupFilename)
+
+	data, err := ioutil.ReadFile(configFile)
+	if err != nil {
+		return fmt.Errorf("failed to read config for backup: %w", err)
+	}
+
+	if err := ioutil.WriteFile(backupFile, data, 0644); err != nil {
+		return fmt.Errorf("failed to write backup file: %w", err)
+	}
+
+	// Cleanup old backups
+	cm.cleanupOldBackups(pluginID)
+
+	return nil
+}
+
+// cleanupOldBackups removes old backup files, keeping only maxBackups
+func (cm *ConfigManager) cleanupOldBackups(pluginID string) {
+	pattern := filepath.Join(cm.backupDir, fmt.Sprintf("%s_*.json.bak", pluginID))
+	files, err := filepath.Glob(pattern)
+	if err != nil {
+		return
+	}
+
+	if len(files) > cm.maxBackups {
+		// Sort by modification time and remove oldest
+		for i := 0; i < len(files)-cm.maxBackups; i++ {
+			os.Remove(files[i])
+		}
+	}
+}
+
+// LoadAllConfigs loads all configurations from the config directory
+func (cm *ConfigManager) LoadAllConfigs() error {
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+
+	files, err := filepath.Glob(filepath.Join(cm.configDir, "*.json"))
+	if err != nil {
+		return fmt.Errorf("failed to list config files: %w", err)
+	}
+
+	for _, file := range files {
+		filename := filepath.Base(file)
+		pluginID := filename[:len(filename)-5] // Remove .json extension
+
+		if pluginID == "plugins" {
+			continue // Skip main config file
+		}
+
+		data, err := ioutil.ReadFile(file)
+		if err != nil {
+			continue
+		}
+
+		var configData map[string]interface{}
+		if err := json.Unmarshal(data, &configData); err != nil {
+			continue
+		}
+
+		// Basic parsing (simplified)
+		config := &PluginConfig{
+			PluginID:    pluginID,
+			Properties:  make(map[string]string),
+			JobTypes:    make(map[string]*JobTypeConfig),
+			Environment: make(map[string]string),
+		}
+
+		cm.pluginConfigs[pluginID] = config
+		cm.configVersions[pluginID] = 1
+		cm.lastModified[pluginID] = time.Now()
+	}
+
+	return nil
+}
+
+// ExportConfigs exports all configurations to a JSON file
+func (cm *ConfigManager) ExportConfigs() (string, error) {
+	cm.mu.RLock()
+	defer cm.mu.RUnlock()
+
+	exportData := make(map[string]interface{})
+	for pluginID, config := range cm.pluginConfigs {
+		exportData[pluginID] = config
+	}
+
+	data, err := json.MarshalIndent(exportData, "", "  ")
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal configs: %w", err)
+	}
+
+	return string(data), nil
+}
+
+// ImportConfigs imports configurations from a JSON string
+func (cm *ConfigManager) ImportConfigs(jsonData string) error {
+	var importData map[string]interface{}
+	if err := json.Unmarshal([]byte(jsonData), &importData); err != nil {
+		return fmt.Errorf("failed to unmarshal import data: %w", err)
+	}
+
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+
+	for pluginID, configData := range importData {
+		if _, ok := configData.(map[string]interface{}); ok {
+			config := &PluginConfig{
+				PluginID:    pluginID,
+				Properties:  make(map[string]string),
+				JobTypes:    make(map[string]*JobTypeConfig),
+				Environment: make(map[string]string),
+			}
+			cm.pluginConfigs[pluginID] = config
+			cm.configVersions[pluginID]++
+			cm.lastModified[pluginID] = time.Now()
+		}
+	}
+
+	return nil
+}

weed/admin/plugin/dispatcher.go

@@ -0,0 +1,382 @@
+package plugin
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+// Dispatcher orchestrates job detection scheduling and dispatch
+type Dispatcher struct {
+	mu                        sync.RWMutex
+	registry                  *Registry
+	queue                     *JobQueue
+	detectionSchedules        map[string]*DetectionSchedule
+	jobTypeStateManagement    map[string]*JobTypeState
+	lastDetectionTime         map[string]time.Time
+	detectionConcurrencyLimit map[string]int
+}
+
+// DetectionSchedule holds scheduling information for a detection type
+type DetectionSchedule struct {
+	DetectionType      string
+	Interval           time.Duration
+	LastExecuted       time.Time
+	NextExecutionTime  time.Time
+	ExecutionCount     int64
+	FailureCount       int64
+	AverageExecutionMs float64
+}
+
+// JobTypeState manages state for a specific job type
+type JobTypeState struct {
+	JobType             string
+	mu                  sync.RWMutex
+	ActiveCount         int
+	MaxConcurrent       int
+	PendingCount        int
+	CompletedCount      int
+	FailedCount         int
+	LastError           string
+	LastExecutionTime   time.Time
+	AverageExecutionMs  float64
+	ExecutionHistory    []time.Duration
+	MaxHistorySize      int
+}
+
+// NewDispatcher creates a new job dispatcher
+func NewDispatcher(registry *Registry, queue *JobQueue) *Dispatcher {
+	return &Dispatcher{
+		registry:                  registry,
+		queue:                     queue,
+		detectionSchedules:        make(map[string]*DetectionSchedule),
+		jobTypeStateManagement:    make(map[string]*JobTypeState),
+		lastDetectionTime:         make(map[string]time.Time),
+		detectionConcurrencyLimit: make(map[string]int),
+	}
+}
+
+// RegisterDetectionType registers a detection type with scheduling info
+func (d *Dispatcher) RegisterDetectionType(detectionType string, interval time.Duration, maxConcurrent int) error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	if _, exists := d.detectionSchedules[detectionType]; exists {
+		return fmt.Errorf("detection type %s already registered", detectionType)
+	}
+
+	d.detectionSchedules[detectionType] = &DetectionSchedule{
+		DetectionType:     detectionType,
+		Interval:          interval,
+		NextExecutionTime: time.Now(),
+	}
+
+	d.detectionConcurrencyLimit[detectionType] = maxConcurrent
+	d.jobTypeStateManagement[detectionType] = &JobTypeState{
+		JobType:        detectionType,
+		MaxConcurrent:  maxConcurrent,
+		MaxHistorySize: 100,
+		ExecutionHistory: make([]time.Duration, 0, 100),
+	}
+
+	return nil
+}
+
+// UnregisterDetectionType removes a detection type
+func (d *Dispatcher) UnregisterDetectionType(detectionType string) error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	if _, exists := d.detectionSchedules[detectionType]; !exists {
+		return fmt.Errorf("detection type %s not found", detectionType)
+	}
+
+	delete(d.detectionSchedules, detectionType)
+	delete(d.detectionConcurrencyLimit, detectionType)
+	delete(d.jobTypeStateManagement, detectionType)
+	delete(d.lastDetectionTime, detectionType)
+
+	return nil
+}
+
+// ScheduleDetections checks and schedules detection jobs that are due
+func (d *Dispatcher) ScheduleDetections() []string {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	var scheduledJobs []string
+	now := time.Now()
+
+	for detectionType, schedule := range d.detectionSchedules {
+		if now.After(schedule.NextExecutionTime) {
+			// Check if we haven't exceeded concurrency limit
+			state := d.jobTypeStateManagement[detectionType]
+			state.mu.RLock()
+			activeCount := state.ActiveCount
+			maxConcurrent := state.MaxConcurrent
+			state.mu.RUnlock()
+
+			if activeCount >= maxConcurrent {
+				continue // Skip this detection type for now
+			}
+
+			// Create and enqueue job
+			jobID := fmt.Sprintf("det-%s-%d", detectionType, now.UnixNano())
+			job := &Job{
+				ID:        jobID,
+				Type:      detectionType,
+				State:     JobStatePending,
+				CreatedAt: now,
+			}
+
+			if err := d.queue.Enqueue(job); err != nil {
+				continue
+			}
+
+			// Update schedule
+			schedule.NextExecutionTime = now.Add(schedule.Interval)
+			schedule.ExecutionCount++
+			d.lastDetectionTime[detectionType] = now
+
+			scheduledJobs = append(scheduledJobs, jobID)
+
+			// Update state
+			state.mu.Lock()
+			state.PendingCount++
+			state.mu.Unlock()
+		}
+	}
+
+	return scheduledJobs
+}
+
+// DispatchJob assigns a job to an available plugin
+func (d *Dispatcher) DispatchJob(job *Job) (string, error) {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+
+	// Find plugins capable of handling this job type
+	plugins := d.registry.GetPluginsByCapability(job.Type)
+	if len(plugins) == 0 {
+		return "", fmt.Errorf("no plugins available for job type %s", job.Type)
+	}
+
+	// Find least loaded available plugin
+	var selectedPlugin *ConnectedPlugin
+	minLoad := int(^uint32(0) >> 1)
+
+	for _, plugin := range plugins {
+		if plugin.IsHealthy(30 * time.Second) {
+			plugin.mu.RLock()
+			if plugin.ActiveJobs < plugin.MaxConcurrentJobs && plugin.ActiveJobs < minLoad {
+				selectedPlugin = plugin
+				minLoad = plugin.ActiveJobs
+			}
+			plugin.mu.RUnlock()
+		}
+	}
+
+	if selectedPlugin == nil {
+		return "", fmt.Errorf("no healthy plugins available for job type %s", job.Type)
+	}
+
+	// Assign job to plugin
+	job.PluginID = selectedPlugin.ID
+	job.SetState(JobStateScheduled)
+	selectedPlugin.IncActiveJobs()
+
+	// Update job type state
+	state := d.jobTypeStateManagement[job.Type]
+	state.mu.Lock()
+	state.ActiveCount++
+	state.PendingCount--
+	state.mu.Unlock()
+
+	return selectedPlugin.ID, nil
+}
+
+// CompleteJob marks a job as completed
+func (d *Dispatcher) CompleteJob(job *Job, result *JobResult) error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	job.Result = result
+	job.SetState(JobStateCompleted)
+
+	// Update plugin
+	if plugin, err := d.registry.GetPlugin(job.PluginID); err == nil {
+		plugin.DecActiveJobs()
+	}
+
+	// Update state
+	if state, exists := d.jobTypeStateManagement[job.Type]; exists {
+		state.mu.Lock()
+		state.ActiveCount--
+		state.CompletedCount++
+		if job.ExecutionTime > 0 {
+			state.ExecutionHistory = append(state.ExecutionHistory, job.ExecutionTime)
+			if len(state.ExecutionHistory) > state.MaxHistorySize {
+				state.ExecutionHistory = state.ExecutionHistory[1:]
+			}
+			d.updateAverageExecutionTime(state)
+		}
+		state.LastExecutionTime = time.Now()
+		state.mu.Unlock()
+	}
+
+	// Update detection schedule if applicable
+	if schedule, exists := d.detectionSchedules[job.Type]; exists {
+		schedule.LastExecuted = time.Now()
+	}
+
+	// Record execution
+	record := &ExecutionRecord{
+		JobID:       job.ID,
+		JobType:     job.Type,
+		PluginID:    job.PluginID,
+		State:       job.State,
+		CreatedAt:   job.CreatedAt,
+		StartedAt:   job.StartedAt,
+		CompletedAt: job.CompletedAt,
+		Result:      result,
+	}
+	d.queue.RecordExecution(record)
+
+	return nil
+}
+
+// FailJob marks a job as failed
+func (d *Dispatcher) FailJob(job *Job, errorMsg string) error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	job.LastError = errorMsg
+	job.SetState(JobStateFailed)
+
+	// Update plugin
+	if plugin, err := d.registry.GetPlugin(job.PluginID); err == nil {
+		plugin.DecActiveJobs()
+	}
+
+	// Update state
+	if state, exists := d.jobTypeStateManagement[job.Type]; exists {
+		state.mu.Lock()
+		state.ActiveCount--
+		state.FailedCount++
+		state.LastError = errorMsg
+		state.LastExecutionTime = time.Now()
+		state.mu.Unlock()
+	}
+
+	// Update detection schedule
+	if schedule, exists := d.detectionSchedules[job.Type]; exists {
+		schedule.FailureCount++
+		schedule.LastExecuted = time.Now()
+	}
+
+	// Record execution
+	record := &ExecutionRecord{
+		JobID:       job.ID,
+		JobType:     job.Type,
+		PluginID:    job.PluginID,
+		State:       job.State,
+		CreatedAt:   job.CreatedAt,
+		StartedAt:   job.StartedAt,
+		CompletedAt: job.CompletedAt,
+		LastError:   errorMsg,
+	}
+	d.queue.RecordExecution(record)
+
+	return nil
+}
+
+// updateAverageExecutionTime recalculates average execution time from history
+func (d *Dispatcher) updateAverageExecutionTime(state *JobTypeState) {
+	if len(state.ExecutionHistory) == 0 {
+		state.AverageExecutionMs = 0
+		return
+	}
+
+	var total int64
+	for _, duration := range state.ExecutionHistory {
+		total += duration.Milliseconds()
+	}
+	state.AverageExecutionMs = float64(total) / float64(len(state.ExecutionHistory))
+}
+
+// GetJobTypeState returns the state for a specific job type
+func (d *Dispatcher) GetJobTypeState(jobType string) *JobTypeState {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+
+	if state, exists := d.jobTypeStateManagement[jobType]; exists {
+		return state
+	}
+	return nil
+}
+
+// GetAllJobTypeStates returns all job type states
+func (d *Dispatcher) GetAllJobTypeStates() map[string]*JobTypeState {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+
+	result := make(map[string]*JobTypeState)
+	for jobType, state := range d.jobTypeStateManagement {
+		result[jobType] = state
+	}
+	return result
+}
+
+// GetDetectionSchedule returns the schedule for a detection type
+func (d *Dispatcher) GetDetectionSchedule(detectionType string) *DetectionSchedule {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+
+	if schedule, exists := d.detectionSchedules[detectionType]; exists {
+		return schedule
+	}
+	return nil
+}
+
+// GetDueDetections returns all detection types that are due for execution
+func (d *Dispatcher) GetDueDetections() []string {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+
+	var due []string
+	now := time.Now()
+
+	for detectionType, schedule := range d.detectionSchedules {
+		if now.After(schedule.NextExecutionTime) {
+			due = append(due, detectionType)
+		}
+	}
+
+	return due
+}
+
+// GetDispatcherStats returns overall dispatcher statistics
+func (d *Dispatcher) GetDispatcherStats() map[string]interface{} {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+
+	totalActive := 0
+	totalCompleted := 0
+	totalFailed := 0
+
+	for _, state := range d.jobTypeStateManagement {
+		state.mu.RLock()
+		totalActive += state.ActiveCount
+		totalCompleted += state.CompletedCount
+		totalFailed += state.FailedCount
+		state.mu.RUnlock()
+	}
+
+	return map[string]interface{}{
+		"detection_types_registered": len(d.detectionSchedules),
+		"total_active_jobs":          totalActive,
+		"total_completed_jobs":       totalCompleted,
+		"total_failed_jobs":          totalFailed,
+		"job_type_states":            len(d.jobTypeStateManagement),
+	}
+}

weed/admin/plugin/grpc_server.go

@@ -0,0 +1,457 @@
+package plugin
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+)
+
+// GRPCServer implements the plugin service gRPC handlers
+type GRPCServer struct {
+	mu            sync.RWMutex
+	registry      *Registry
+	queue         *JobQueue
+	dispatcher    *Dispatcher
+	configMgr     *ConfigManager
+	streamMu      sync.RWMutex
+	activeStreams map[string][]chan interface{}
+	plugin_pb.UnimplementedPluginServiceServer
+	plugin_pb.UnimplementedAdminQueryServiceServer
+	plugin_pb.UnimplementedAdminCommandServiceServer
+}
+
+// NewGRPCServer creates a new gRPC server
+func NewGRPCServer(registry *Registry, queue *JobQueue, dispatcher *Dispatcher, configMgr *ConfigManager) *GRPCServer {
+	return &GRPCServer{
+		registry:      registry,
+		queue:         queue,
+		dispatcher:    dispatcher,
+		configMgr:     configMgr,
+		activeStreams: make(map[string][]chan interface{}),
+	}
+}
+
+// Connect registers a plugin with the master
+func (gs *GRPCServer) Connect(ctx context.Context, req *plugin_pb.PluginConnectRequest) (*plugin_pb.PluginConnectResponse, error) {
+	if req.PluginId == "" {
+		return nil, fmt.Errorf("plugin_id is required")
+	}
+
+	// Create ConnectedPlugin instance
+	plugin := &ConnectedPlugin{
+		ID:                  req.PluginId,
+		Name:                req.PluginName,
+		Version:             req.Version,
+		Status:              "CONNECTED",
+		Capabilities:        req.Capabilities,
+		MaxConcurrentJobs:   int(req.MaxConcurrentJobs),
+		ConnectedAt:         time.Now(),
+		LastHeartbeat:       time.Now(),
+		Metadata:            req.Metadata,
+		HealthCheckInterval: 30 * time.Second,
+		JobTimeout:          5 * time.Minute,
+	}
+
+	// Register plugin
+	if err := gs.registry.RegisterPlugin(plugin); err != nil {
+		return nil, fmt.Errorf("failed to register plugin: %w", err)
+	}
+
+	// Load or create configuration
+	config, err := gs.configMgr.LoadConfig(req.PluginId)
+	if err != nil {
+		// Create default config
+		config = &PluginConfig{
+			PluginID:            req.PluginId,
+			Properties:          make(map[string]string),
+			JobTypes:            make(map[string]*JobTypeConfig),
+			MaxRetries:          3,
+			HealthCheckInterval: 30 * time.Second,
+			JobTimeout:          5 * time.Minute,
+			Environment:         make(map[string]string),
+		}
+		gs.configMgr.SaveConfig(config, false)
+	}
+
+	// Build response
+	pbConfig := &plugin_pb.PluginConfig{
+		PluginId:    config.PluginID,
+		Properties:  config.Properties,
+		MaxRetries:  int32(config.MaxRetries),
+		Environment: config.Environment,
+	}
+
+	response := &plugin_pb.PluginConnectResponse{
+		Success:       true,
+		Message:       "Plugin registered successfully",
+		MasterId:      "master-1",
+		Config:        pbConfig,
+		AssignedTypes: req.Capabilities,
+	}
+
+	return response, nil
+}
+
+// ExecuteJob processes a detection or maintenance job
+func (gs *GRPCServer) ExecuteJob(ctx context.Context, req *plugin_pb.ExecuteJobRequest) (*plugin_pb.ExecuteJobResponse, error) {
+	if req.JobId == "" || req.JobType == "" {
+		return nil, fmt.Errorf("job_id and job_type are required")
+	}
+
+	response := &plugin_pb.ExecuteJobResponse{
+		JobId:   req.JobId,
+		Status:  plugin_pb.ExecutionStatus_EXECUTION_STATUS_ACCEPTED,
+		Message: "Job accepted for execution",
+	}
+
+	return response, nil
+}
+
+// ReportHealth processes health reports from plugins
+func (gs *GRPCServer) ReportHealth(ctx context.Context, report *plugin_pb.HealthReport) (*plugin_pb.HealthReportResponse, error) {
+	if report.PluginId == "" {
+		return nil, fmt.Errorf("plugin_id is required")
+	}
+
+	// Update heartbeat
+	if err := gs.registry.UpdateHeartbeat(report.PluginId); err != nil {
+		return nil, fmt.Errorf("plugin not found: %w", err)
+	}
+
+	// Update plugin stats
+	if plugin, err := gs.registry.GetPlugin(report.PluginId); err == nil {
+		plugin.mu.Lock()
+		plugin.ActiveJobs = int(report.ActiveJobs)
+		plugin.CPUUsagePercent = float64(report.CpuPercent)
+		plugin.MemoryUsageBytes = report.MemoryBytes
+		plugin.mu.Unlock()
+	}
+
+	return &plugin_pb.HealthReportResponse{
+		Acknowledged: true,
+		Feedback:     "Health report received",
+	}, nil
+}
+
+// GetConfig retrieves the latest configuration
+func (gs *GRPCServer) GetConfig(ctx context.Context, req *plugin_pb.GetConfigRequest) (*plugin_pb.GetConfigResponse, error) {
+	if req.PluginId == "" {
+		return nil, fmt.Errorf("plugin_id is required")
+	}
+
+	config, exists := gs.configMgr.GetConfig(req.PluginId)
+	if !exists {
+		return nil, fmt.Errorf("config not found for plugin: %s", req.PluginId)
+	}
+
+	pbConfig := &plugin_pb.PluginConfig{
+		PluginId:    config.PluginID,
+		Properties:  config.Properties,
+		MaxRetries:  int32(config.MaxRetries),
+		Environment: config.Environment,
+	}
+
+	response := &plugin_pb.GetConfigResponse{
+		Config:  pbConfig,
+		Version: gs.configMgr.GetVersion(req.PluginId),
+	}
+
+	return response, nil
+}
+
+// SubmitResult sends job execution results back to master
+func (gs *GRPCServer) SubmitResult(ctx context.Context, req *plugin_pb.JobResultRequest) (*plugin_pb.JobResultResponse, error) {
+	if req.JobId == "" {
+		return nil, fmt.Errorf("job_id is required")
+	}
+
+	actions := []string{}
+
+	// Process results based on job status
+	switch req.Status {
+	case plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED:
+		actions = append(actions, "ARCHIVED")
+	case plugin_pb.ExecutionStatus_EXECUTION_STATUS_FAILED:
+		actions = append(actions, "RETRY", "NOTIFY_ADMIN")
+	}
+
+	response := &plugin_pb.JobResultResponse{
+		Acknowledged:  true,
+		ActionsToTake: actions,
+	}
+
+	return response, nil
+}
+
+// GetPluginStats returns statistics for all connected plugins
+func (gs *GRPCServer) GetPluginStats(ctx context.Context, req *plugin_pb.GetPluginStatsRequest) (*plugin_pb.GetPluginStatsResponse, error) {
+	response := &plugin_pb.GetPluginStatsResponse{
+		Stats: []*plugin_pb.PluginStats{},
+	}
+
+	var plugins []*ConnectedPlugin
+	if req.PluginId != "" {
+		if plugin, err := gs.registry.GetPlugin(req.PluginId); err == nil {
+			plugins = append(plugins, plugin)
+		}
+	} else {
+		plugins = gs.registry.ListPlugins(false)
+	}
+
+	for _, plugin := range plugins {
+		stat := &plugin_pb.PluginStats{
+			PluginId:           plugin.ID,
+			Status:             plugin.Status,
+			ActiveJobs:         int32(plugin.ActiveJobs),
+			CompletedJobs:      int32(plugin.CompletedJobs),
+			FailedJobs:         int32(plugin.FailedJobs),
+			TotalDetections:    plugin.TotalDetections,
+			AvgExecutionTimeMs: float32(plugin.AvgExecutionTimeMs),
+			CpuUsagePercent:    float32(plugin.CPUUsagePercent),
+			MemoryUsageBytes:   plugin.MemoryUsageBytes,
+			UptimeSeconds:      int32(time.Since(plugin.ConnectedAt).Seconds()),
+		}
+		response.Stats = append(response.Stats, stat)
+	}
+
+	return response, nil
+}
+
+// ListPlugins returns information about all registered plugins
+func (gs *GRPCServer) ListPlugins(ctx context.Context, req *plugin_pb.ListPluginsRequest) (*plugin_pb.ListPluginsResponse, error) {
+	response := &plugin_pb.ListPluginsResponse{
+		Plugins: []*plugin_pb.PluginInfo{},
+	}
+
+	plugins := gs.registry.ListPlugins(!req.IncludeDisabled)
+
+	for _, plugin := range plugins {
+		// Filter by capability if specified
+		if len(req.FilterByCapability) > 0 {
+			hasCapability := false
+			for _, filterCap := range req.FilterByCapability {
+				for _, cap := range plugin.Capabilities {
+					if cap == filterCap {
+						hasCapability = true
+						break
+					}
+				}
+				if hasCapability {
+					break
+				}
+			}
+			if !hasCapability {
+				continue
+			}
+		}
+
+		info := &plugin_pb.PluginInfo{
+			PluginId:          plugin.ID,
+			Name:              plugin.Name,
+			Version:           plugin.Version,
+			Status:            plugin.Status,
+			Capabilities:      plugin.Capabilities,
+			MaxConcurrentJobs: int32(plugin.MaxConcurrentJobs),
+			ActiveJobs:        int32(plugin.ActiveJobs),
+			Metadata:          plugin.Metadata,
+		}
+		response.Plugins = append(response.Plugins, info)
+	}
+
+	return response, nil
+}
+
+// ListJobs returns current and historical job information
+func (gs *GRPCServer) ListJobs(ctx context.Context, req *plugin_pb.ListJobsRequest) (*plugin_pb.ListJobsResponse, error) {
+	response := &plugin_pb.ListJobsResponse{
+		Jobs: []*plugin_pb.JobInfo{},
+	}
+
+	var records []*ExecutionRecord
+	limit := int(req.Limit)
+	if limit <= 0 {
+		limit = 100
+	}
+
+	if req.PluginId != "" {
+		records = gs.queue.GetHistoryForPlugin(req.PluginId, limit)
+	} else {
+		records = gs.queue.GetHistory(limit)
+	}
+
+	for _, record := range records {
+		info := &plugin_pb.JobInfo{
+			JobId:      record.JobID,
+			JobType:    record.JobType,
+			PluginId:   record.PluginID,
+			RetryCount: int32(record.RetryCount),
+			LastError:  record.LastError,
+		}
+		response.Jobs = append(response.Jobs, info)
+	}
+
+	response.TotalCount = int32(gs.queue.HistorySize())
+	return response, nil
+}
+
+// GetJobStatus returns detailed status of a specific job
+func (gs *GRPCServer) GetJobStatus(ctx context.Context, req *plugin_pb.GetJobStatusRequest) (*plugin_pb.GetJobStatusResponse, error) {
+	if req.JobId == "" {
+		return nil, fmt.Errorf("job_id is required")
+	}
+
+	// Search in queue history
+	records := gs.queue.GetHistory(10000)
+	for _, record := range records {
+		if record.JobID == req.JobId {
+			response := &plugin_pb.GetJobStatusResponse{
+				DetailedStatus: record.State.String(),
+			}
+			return response, nil
+		}
+	}
+
+	return nil, fmt.Errorf("job not found: %s", req.JobId)
+}
+
+// GetPluginLogs returns logs from a specific plugin (stub implementation)
+func (gs *GRPCServer) GetPluginLogs(ctx context.Context, req *plugin_pb.GetPluginLogsRequest) (*plugin_pb.GetPluginLogsResponse, error) {
+	response := &plugin_pb.GetPluginLogsResponse{
+		Entries: []*plugin_pb.LogEntry{},
+	}
+	return response, nil
+}
+
+// SaveConfig persists plugin configuration
+func (gs *GRPCServer) SaveConfig(ctx context.Context, req *plugin_pb.SaveConfigRequest) (*plugin_pb.SaveConfigResponse, error) {
+	if req.Config == nil {
+		return nil, fmt.Errorf("config is required")
+	}
+
+	// Convert from protobuf config to internal config
+	config := &PluginConfig{
+		PluginID:    req.Config.PluginId,
+		Properties:  req.Config.Properties,
+		MaxRetries:  int(req.Config.MaxRetries),
+		Environment: req.Config.Environment,
+		JobTypes:    make(map[string]*JobTypeConfig),
+	}
+
+	if err := gs.configMgr.SaveConfig(config, req.BackupExisting); err != nil {
+		return nil, fmt.Errorf("failed to save config: %w", err)
+	}
+
+	response := &plugin_pb.SaveConfigResponse{
+		Success:       true,
+		Message:       "Configuration saved successfully",
+		ConfigVersion: gs.configMgr.GetVersion(req.Config.PluginId),
+	}
+
+	return response, nil
+}
+
+// ReloadConfig reloads configuration without restarting
+func (gs *GRPCServer) ReloadConfig(ctx context.Context, req *plugin_pb.ReloadConfigRequest) (*plugin_pb.ReloadConfigResponse, error) {
+	if req.PluginId == "" {
+		return nil, fmt.Errorf("plugin_id is required")
+	}
+
+	if _, err := gs.configMgr.LoadConfig(req.PluginId); err != nil {
+		return nil, fmt.Errorf("failed to reload config: %w", err)
+	}
+
+	response := &plugin_pb.ReloadConfigResponse{
+		Success: true,
+		Message: "Configuration reloaded successfully",
+	}
+
+	return response, nil
+}
+
+// EnablePlugin enables a specific plugin
+func (gs *GRPCServer) EnablePlugin(ctx context.Context, req *plugin_pb.EnablePluginRequest) (*plugin_pb.EnablePluginResponse, error) {
+	if err := gs.registry.UpdatePluginStatus(req.PluginId, "ENABLED"); err != nil {
+		return nil, fmt.Errorf("failed to enable plugin: %w", err)
+	}
+
+	response := &plugin_pb.EnablePluginResponse{
+		Success: true,
+		Message: "Plugin enabled successfully",
+	}
+
+	return response, nil
+}
+
+// DisablePlugin disables a specific plugin
+func (gs *GRPCServer) DisablePlugin(ctx context.Context, req *plugin_pb.DisablePluginRequest) (*plugin_pb.DisablePluginResponse, error) {
+	if err := gs.registry.UpdatePluginStatus(req.PluginId, "DISABLED"); err != nil {
+		return nil, fmt.Errorf("failed to disable plugin: %w", err)
+	}
+
+	response := &plugin_pb.DisablePluginResponse{
+		Success: true,
+		Message: "Plugin disabled successfully",
+	}
+
+	return response, nil
+}
+
+// TriggerDetection manually triggers a detection for specific types
+func (gs *GRPCServer) TriggerDetection(ctx context.Context, req *plugin_pb.TriggerDetectionRequest) (*plugin_pb.TriggerDetectionResponse, error) {
+	response := &plugin_pb.TriggerDetectionResponse{
+		Success:         true,
+		TriggeredJobIds: []string{},
+	}
+
+	for _, detectionType := range req.DetectionTypes {
+		jobID := fmt.Sprintf("trig-%s-%d", detectionType, time.Now().UnixNano())
+		job := &Job{
+			ID:        jobID,
+			Type:      detectionType,
+			State:     JobStatePending,
+			CreatedAt: time.Now(),
+		}
+
+		if err := gs.queue.Enqueue(job); err != nil {
+			continue
+		}
+		response.TriggeredJobIds = append(response.TriggeredJobIds, jobID)
+	}
+
+	return response, nil
+}
+
+// CancelJob cancels a running job
+func (gs *GRPCServer) CancelJob(ctx context.Context, req *plugin_pb.CancelJobRequest) (*plugin_pb.CancelJobResponse, error) {
+	if req.JobId == "" {
+		return nil, fmt.Errorf("job_id is required")
+	}
+
+	if gs.queue.RemoveJob(req.JobId) {
+		return &plugin_pb.CancelJobResponse{
+			Success: true,
+			Message: "Job cancelled successfully",
+		}, nil
+	}
+
+	return &plugin_pb.CancelJobResponse{
+		Success: false,
+		Message: "Job not found or already completed",
+	}, nil
+}
+
+// PurgeHistory clears job history
+func (gs *GRPCServer) PurgeHistory(ctx context.Context, req *plugin_pb.PurgeHistoryRequest) (*plugin_pb.PurgeHistoryResponse, error) {
+	beforeTime := time.Unix(0, req.BeforeTimestampMs*1000000)
+	deleted := gs.queue.PurgeOldHistory(beforeTime)
+
+	response := &plugin_pb.PurgeHistoryResponse{
+		Success:        true,
+		RecordsDeleted: int32(deleted),
+	}
+
+	return response, nil
+}

weed/admin/plugin/job_queue.go

@@ -0,0 +1,298 @@
+package plugin
+
+import (
+	"container/heap"
+	"fmt"
+	"sync"
+	"time"
+)
+
+// JobQueue manages job queueing with priority, deduplication, retry and history
+type JobQueue struct {
+	mu              sync.RWMutex
+	priorityQueue   *PriorityQueue
+	seenJobs        map[string]bool // For deduplication
+	jobHistory      []*ExecutionRecord
+	maxHistorySize  int
+	deduplicationTTL time.Duration
+	lastSeenJob     map[string]time.Time
+}
+
+// PriorityQueue implements heap.Interface for job ordering
+type PriorityQueue []*Job
+
+func (pq PriorityQueue) Len() int { return len(pq) }
+
+func (pq PriorityQueue) Less(i, j int) bool {
+	// Higher priority jobs come first
+	if pq[i].Type != pq[j].Type {
+		return pq[i].Type < pq[j].Type
+	}
+	// If same type, earlier creation time comes first
+	return pq[i].CreatedAt.Before(pq[j].CreatedAt)
+}
+
+func (pq PriorityQueue) Swap(i, j int) {
+	pq[i], pq[j] = pq[j], pq[i]
+}
+
+func (pq *PriorityQueue) Push(x interface{}) {
+	*pq = append(*pq, x.(*Job))
+}
+
+func (pq *PriorityQueue) Pop() interface{} {
+	old := *pq
+	n := len(old)
+	item := old[n-1]
+	*pq = old[0 : n-1]
+	return item
+}
+
+// NewJobQueue creates a new job queue
+func NewJobQueue(maxHistorySize int, deduplicationTTL time.Duration) *JobQueue {
+	jq := &JobQueue{
+		priorityQueue:    &PriorityQueue{},
+		seenJobs:         make(map[string]bool),
+		jobHistory:       make([]*ExecutionRecord, 0, maxHistorySize),
+		maxHistorySize:   maxHistorySize,
+		deduplicationTTL: deduplicationTTL,
+		lastSeenJob:      make(map[string]time.Time),
+	}
+	heap.Init(jq.priorityQueue)
+	return jq
+}
+
+// Enqueue adds a job to the queue with deduplication
+func (jq *JobQueue) Enqueue(job *Job) error {
+	jq.mu.Lock()
+	defer jq.mu.Unlock()
+
+	// Check for duplicates within TTL window
+	if lastSeen, exists := jq.lastSeenJob[job.ID]; exists {
+		if time.Since(lastSeen) < jq.deduplicationTTL {
+			return fmt.Errorf("job %s already enqueued recently", job.ID)
+		}
+	}
+
+	job.SetState(JobStatePending)
+	heap.Push(jq.priorityQueue, job)
+	jq.seenJobs[job.ID] = true
+	jq.lastSeenJob[job.ID] = time.Now()
+
+	return nil
+}
+
+// Dequeue retrieves the next job from the queue
+func (jq *JobQueue) Dequeue() *Job {
+	jq.mu.Lock()
+	defer jq.mu.Unlock()
+
+	if jq.priorityQueue.Len() == 0 {
+		return nil
+	}
+
+	job := heap.Pop(jq.priorityQueue).(*Job)
+	return job
+}
+
+// Peek returns the next job without removing it
+func (jq *JobQueue) Peek() *Job {
+	jq.mu.RLock()
+	defer jq.mu.RUnlock()
+
+	if jq.priorityQueue.Len() == 0 {
+		return nil
+	}
+
+	return (*jq.priorityQueue)[0]
+}
+
+// Size returns the current queue size
+func (jq *JobQueue) Size() int {
+	jq.mu.RLock()
+	defer jq.mu.RUnlock()
+	return jq.priorityQueue.Len()
+}
+
+// RecordExecution adds an execution record to history
+func (jq *JobQueue) RecordExecution(record *ExecutionRecord) {
+	jq.mu.Lock()
+	defer jq.mu.Unlock()
+
+	jq.jobHistory = append(jq.jobHistory, record)
+
+	// Keep history size bounded
+	if len(jq.jobHistory) > jq.maxHistorySize {
+		// Remove oldest entries
+		removeCount := len(jq.jobHistory) - jq.maxHistorySize
+		jq.jobHistory = jq.jobHistory[removeCount:]
+	}
+}
+
+// GetHistory returns job execution history
+func (jq *JobQueue) GetHistory(limit int) []*ExecutionRecord {
+	jq.mu.RLock()
+	defer jq.mu.RUnlock()
+
+	if limit <= 0 || limit > len(jq.jobHistory) {
+		limit = len(jq.jobHistory)
+	}
+
+	// Return the most recent entries
+	startIdx := len(jq.jobHistory) - limit
+	if startIdx < 0 {
+		startIdx = 0
+	}
+
+	result := make([]*ExecutionRecord, limit)
+	copy(result, jq.jobHistory[startIdx:])
+	return result
+}
+
+// GetHistoryForPlugin returns history for a specific plugin
+func (jq *JobQueue) GetHistoryForPlugin(pluginID string, limit int) []*ExecutionRecord {
+	jq.mu.RLock()
+	defer jq.mu.RUnlock()
+
+	var result []*ExecutionRecord
+	for i := len(jq.jobHistory) - 1; i >= 0 && len(result) < limit; i-- {
+		if jq.jobHistory[i].PluginID == pluginID {
+			result = append(result, jq.jobHistory[i])
+		}
+	}
+	return result
+}
+
+// GetHistoryForJobType returns history for a specific job type
+func (jq *JobQueue) GetHistoryForJobType(jobType string, limit int) []*ExecutionRecord {
+	jq.mu.RLock()
+	defer jq.mu.RUnlock()
+
+	var result []*ExecutionRecord
+	for i := len(jq.jobHistory) - 1; i >= 0 && len(result) < limit; i-- {
+		if jq.jobHistory[i].JobType == jobType {
+			result = append(result, jq.jobHistory[i])
+		}
+	}
+	return result
+}
+
+// ClearHistory removes all execution history
+func (jq *JobQueue) ClearHistory() {
+	jq.mu.Lock()
+	defer jq.mu.Unlock()
+	jq.jobHistory = make([]*ExecutionRecord, 0, jq.maxHistorySize)
+}
+
+// PurgeOldHistory removes history entries older than the specified time
+func (jq *JobQueue) PurgeOldHistory(beforeTime time.Time) int {
+	jq.mu.Lock()
+	defer jq.mu.Unlock()
+
+	removed := 0
+	newHistory := make([]*ExecutionRecord, 0)
+
+	for _, record := range jq.jobHistory {
+		if record.CreatedAt.After(beforeTime) {
+			newHistory = append(newHistory, record)
+		} else {
+			removed++
+		}
+	}
+
+	jq.jobHistory = newHistory
+	return removed
+}
+
+// HistorySize returns the number of records in history
+func (jq *JobQueue) HistorySize() int {
+	jq.mu.RLock()
+	defer jq.mu.RUnlock()
+	return len(jq.jobHistory)
+}
+
+// RetryJob re-enqueues a failed job up to maxRetries times
+func (jq *JobQueue) RetryJob(job *Job, maxRetries int) error {
+	jq.mu.Lock()
+	defer jq.mu.Unlock()
+
+	if job.RetryCount >= maxRetries {
+		return fmt.Errorf("job %s exceeded max retries (%d)", job.ID, maxRetries)
+	}
+
+	job.RetryCount++
+	job.SetState(JobStatePending)
+	heap.Push(jq.priorityQueue, job)
+
+	return nil
+}
+
+// GetExecutionStats returns statistics about job executions
+func (jq *JobQueue) GetExecutionStats() map[string]interface{} {
+	jq.mu.RLock()
+	defer jq.mu.RUnlock()
+
+	completed := 0
+	failed := 0
+	totalExecutionTime := int64(0)
+
+	for _, record := range jq.jobHistory {
+		switch record.State {
+		case JobStateCompleted:
+			completed++
+		case JobStateFailed:
+			failed++
+		}
+		if record.CompletedAt != nil && record.StartedAt != nil {
+			totalExecutionTime += record.CompletedAt.Sub(*record.StartedAt).Milliseconds()
+		}
+	}
+
+	avgExecutionTime := int64(0)
+	if completed+failed > 0 {
+		avgExecutionTime = totalExecutionTime / int64(completed+failed)
+	}
+
+	return map[string]interface{}{
+		"total_history":          len(jq.jobHistory),
+		"completed_jobs":         completed,
+		"failed_jobs":            failed,
+		"avg_execution_time_ms":  avgExecutionTime,
+		"current_queue_size":     jq.priorityQueue.Len(),
+	}
+}
+
+// GetQueuedJobs returns all jobs currently in the queue
+func (jq *JobQueue) GetQueuedJobs() []*Job {
+	jq.mu.RLock()
+	defer jq.mu.RUnlock()
+
+	result := make([]*Job, len(*jq.priorityQueue))
+	copy(result, *jq.priorityQueue)
+	return result
+}
+
+// RemoveJob removes a specific job from the queue
+func (jq *JobQueue) RemoveJob(jobID string) bool {
+	jq.mu.Lock()
+	defer jq.mu.Unlock()
+
+	for i, job := range *jq.priorityQueue {
+		if job.ID == jobID {
+			heap.Remove(jq.priorityQueue, i)
+			return true
+		}
+	}
+	return false
+}
+
+// PurgeQueuedJobs clears all pending jobs from the queue
+func (jq *JobQueue) PurgeQueuedJobs() int {
+	jq.mu.Lock()
+	defer jq.mu.Unlock()
+
+	count := jq.priorityQueue.Len()
+	*jq.priorityQueue = PriorityQueue{}
+	heap.Init(jq.priorityQueue)
+	return count
+}

weed/admin/plugin/manager.go

@@ -0,0 +1,498 @@
+package plugin
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+)
+
+// Manager is the main component orchestrating the plugin system
+type Manager struct {
+	mu                sync.RWMutex
+	registry          *Registry
+	queue             *JobQueue
+	dispatcher        *Dispatcher
+	configMgr         *ConfigManager
+	grpcServer        *GRPCServer
+	isRunning         bool
+	stopChan          chan bool
+	scheduleTicker    *time.Ticker
+	healthCheckTicker *time.Ticker
+	detectionTicker   *time.Ticker
+	wg                sync.WaitGroup
+	ctx               context.Context
+	cancel            context.CancelFunc
+	config            *ManagerConfig
+}
+
+// ManagerConfig holds configuration for the plugin manager
+type ManagerConfig struct {
+	ConfigDir              string
+	ScheduleInterval       time.Duration
+	HealthCheckInterval    time.Duration
+	DetectionInterval      time.Duration
+	MaxQueueSize           int
+	MaxHistorySize         int
+	DeduplicationTTL       time.Duration
+	HealthCheckTimeout     time.Duration
+	FailureDetectionWindow time.Duration
+	FailureThreshold       int
+}
+
+// DefaultManagerConfig returns default configuration
+func DefaultManagerConfig(configDir string) *ManagerConfig {
+	return &ManagerConfig{
+		ConfigDir:              configDir,
+		ScheduleInterval:       5 * time.Second,
+		HealthCheckInterval:    30 * time.Second,
+		DetectionInterval:      10 * time.Second,
+		MaxQueueSize:           10000,
+		MaxHistorySize:         5000,
+		DeduplicationTTL:       1 * time.Minute,
+		HealthCheckTimeout:     90 * time.Second,
+		FailureDetectionWindow: 5 * time.Minute,
+		FailureThreshold:       3,
+	}
+}
+
+// NewManager creates a new plugin manager instance
+func NewManager(config *ManagerConfig) (*Manager, error) {
+	if config == nil {
+		return nil, fmt.Errorf("config is required")
+	}
+
+	// Create configuration manager
+	configMgr, err := NewConfigManager(config.ConfigDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create config manager: %w", err)
+	}
+
+	// Create registry
+	registry := NewRegistry(
+		config.HealthCheckTimeout,
+		config.FailureDetectionWindow,
+		config.FailureThreshold,
+	)
+
+	// Create job queue
+	queue := NewJobQueue(config.MaxHistorySize, config.DeduplicationTTL)
+
+	// Create dispatcher
+	dispatcher := NewDispatcher(registry, queue)
+
+	// Create gRPC server
+	grpcServer := NewGRPCServer(registry, queue, dispatcher, configMgr)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	manager := &Manager{
+		registry:   registry,
+		queue:      queue,
+		dispatcher: dispatcher,
+		configMgr:  configMgr,
+		grpcServer: grpcServer,
+		stopChan:   make(chan bool),
+		ctx:        ctx,
+		cancel:     cancel,
+		config:     config,
+	}
+
+	return manager, nil
+}
+
+// Start initializes and starts the plugin manager
+func (m *Manager) Start() error {
+	m.mu.Lock()
+	if m.isRunning {
+		m.mu.Unlock()
+		return fmt.Errorf("manager already running")
+	}
+	m.isRunning = true
+	m.mu.Unlock()
+
+	// Load existing configurations
+	if err := m.configMgr.LoadAllConfigs(); err != nil {
+		m.isRunning = false
+		return fmt.Errorf("failed to load configurations: %w", err)
+	}
+
+	// Start background tasks
+	m.wg.Add(3)
+	go m.schedulerLoop()
+	go m.healthCheckLoop()
+	go m.detectionLoop()
+
+	return nil
+}
+
+// Stop gracefully stops the plugin manager
+func (m *Manager) Stop() error {
+	m.mu.Lock()
+	if !m.isRunning {
+		m.mu.Unlock()
+		return fmt.Errorf("manager not running")
+	}
+	m.isRunning = false
+	m.mu.Unlock()
+
+	// Signal all goroutines to stop
+	m.cancel()
+	close(m.stopChan)
+
+	// Wait for all goroutines to finish
+	done := make(chan struct{})
+	go func() {
+		m.wg.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		return nil
+	case <-time.After(30 * time.Second):
+		return fmt.Errorf("timeout waiting for manager to stop")
+	}
+}
+
+// schedulerLoop periodically schedules detection jobs
+func (m *Manager) schedulerLoop() {
+	defer m.wg.Done()
+
+	m.scheduleTicker = time.NewTicker(m.config.ScheduleInterval)
+	defer m.scheduleTicker.Stop()
+
+	for {
+		select {
+		case <-m.ctx.Done():
+			return
+		case <-m.stopChan:
+			return
+		case <-m.scheduleTicker.C:
+			m.performScheduling()
+		}
+	}
+}
+
+// performScheduling executes the scheduling logic
+func (m *Manager) performScheduling() {
+	scheduledJobs := m.dispatcher.ScheduleDetections()
+	if len(scheduledJobs) > 0 {
+		// Jobs have been queued for processing
+	}
+}
+
+// healthCheckLoop periodically checks plugin health
+func (m *Manager) healthCheckLoop() {
+	defer m.wg.Done()
+
+	m.healthCheckTicker = time.NewTicker(m.config.HealthCheckInterval)
+	defer m.healthCheckTicker.Stop()
+
+	for {
+		select {
+		case <-m.ctx.Done():
+			return
+		case <-m.stopChan:
+			return
+		case <-m.healthCheckTicker.C:
+			m.performHealthCheck()
+		}
+	}
+}
+
+// performHealthCheck checks health of all plugins
+func (m *Manager) performHealthCheck() {
+	plugins := m.registry.ListPlugins(true)
+
+	for _, plugin := range plugins {
+		isHealthy, _ := m.registry.HealthCheck(plugin.ID)
+		if !isHealthy {
+			// Check if exceeded threshold
+			if m.registry.HasExceededFailureThreshold(plugin.ID) {
+				m.registry.UnregisterPlugin(plugin.ID)
+			}
+		}
+	}
+}
+
+// detectionLoop periodically triggers detection execution
+func (m *Manager) detectionLoop() {
+	defer m.wg.Done()
+
+	m.detectionTicker = time.NewTicker(m.config.DetectionInterval)
+	defer m.detectionTicker.Stop()
+
+	for {
+		select {
+		case <-m.ctx.Done():
+			return
+		case <-m.stopChan:
+			return
+		case <-m.detectionTicker.C:
+			m.processDetectionJobs()
+		}
+	}
+}
+
+// processDetectionJobs dequeues and dispatches pending jobs
+func (m *Manager) processDetectionJobs() {
+	for {
+		job := m.queue.Dequeue()
+		if job == nil {
+			break
+		}
+
+		// Dispatch job to available plugin
+		pluginID, err := m.dispatcher.DispatchJob(job)
+		if err != nil {
+			// Requeue job if dispatch failed
+			m.queue.Enqueue(job)
+			break
+		}
+
+		job.PluginID = pluginID
+	}
+}
+
+// IsRunning returns whether the manager is currently running
+func (m *Manager) IsRunning() bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.isRunning
+}
+
+// RegisterDetectionType registers a new detection type
+func (m *Manager) RegisterDetectionType(detectionType string, interval time.Duration, maxConcurrent int) error {
+	return m.dispatcher.RegisterDetectionType(detectionType, interval, maxConcurrent)
+}
+
+// UnregisterDetectionType unregisters a detection type
+func (m *Manager) UnregisterDetectionType(detectionType string) error {
+	return m.dispatcher.UnregisterDetectionType(detectionType)
+}
+
+// GetStats returns overall statistics
+func (m *Manager) GetStats() map[string]interface{} {
+	return map[string]interface{}{
+		"registry":   m.registry.GetStats(),
+		"queue":      m.queue.GetExecutionStats(),
+		"dispatcher": m.dispatcher.GetDispatcherStats(),
+		"running":    m.IsRunning(),
+	}
+}
+
+// GetPluginStats returns statistics for a specific plugin
+func (m *Manager) GetPluginStats(pluginID string) (map[string]interface{}, error) {
+	plugin, err := m.registry.GetPlugin(pluginID)
+	if err != nil {
+		return nil, err
+	}
+
+	plugin.mu.RLock()
+	defer plugin.mu.RUnlock()
+
+	return map[string]interface{}{
+		"id":                      plugin.ID,
+		"name":                    plugin.Name,
+		"version":                 plugin.Version,
+		"status":                  plugin.Status,
+		"capabilities":            plugin.Capabilities,
+		"active_jobs":             plugin.ActiveJobs,
+		"completed_jobs":          plugin.CompletedJobs,
+		"failed_jobs":             plugin.FailedJobs,
+		"total_detections":        plugin.TotalDetections,
+		"avg_execution_time_ms":   plugin.AvgExecutionTimeMs,
+		"cpu_usage_percent":       plugin.CPUUsagePercent,
+		"memory_usage_bytes":      plugin.MemoryUsageBytes,
+		"connected_at":            plugin.ConnectedAt,
+		"last_heartbeat":          plugin.LastHeartbeat,
+		"uptime_seconds":          int(time.Since(plugin.ConnectedAt).Seconds()),
+	}, nil
+}
+
+// ListPlugins returns all registered plugins
+func (m *Manager) ListPlugins(includeUnhealthy bool) []*ConnectedPlugin {
+	return m.registry.ListPlugins(includeUnhealthy)
+}
+
+// ListJobs returns job history
+func (m *Manager) ListJobs(limit int) []*ExecutionRecord {
+	return m.queue.GetHistory(limit)
+}
+
+// ListJobsForPlugin returns jobs for a specific plugin
+func (m *Manager) ListJobsForPlugin(pluginID string, limit int) []*ExecutionRecord {
+	return m.queue.GetHistoryForPlugin(pluginID, limit)
+}
+
+// ListJobsForType returns jobs for a specific type
+func (m *Manager) ListJobsForType(jobType string, limit int) []*ExecutionRecord {
+	return m.queue.GetHistoryForJobType(jobType, limit)
+}
+
+// TriggerDetection manually triggers detection for specific types
+func (m *Manager) TriggerDetection(detectionTypes []string) ([]string, error) {
+	var jobIDs []string
+
+	for _, detectionType := range detectionTypes {
+		jobID := fmt.Sprintf("manual-%s-%d", detectionType, time.Now().UnixNano())
+		job := &Job{
+			ID:        jobID,
+			Type:      detectionType,
+			State:     JobStatePending,
+			CreatedAt: time.Now(),
+		}
+
+		if err := m.queue.Enqueue(job); err != nil {
+			continue
+		}
+		jobIDs = append(jobIDs, jobID)
+	}
+
+	return jobIDs, nil
+}
+
+// GetJobStatus returns the status of a specific job
+func (m *Manager) GetJobStatus(jobID string) (*ExecutionRecord, error) {
+	records := m.queue.GetHistory(10000)
+	for _, record := range records {
+		if record.JobID == jobID {
+			return record, nil
+		}
+	}
+	return nil, fmt.Errorf("job not found: %s", jobID)
+}
+
+// CancelJob cancels a pending or scheduled job
+func (m *Manager) CancelJob(jobID string) error {
+	if !m.queue.RemoveJob(jobID) {
+		return fmt.Errorf("job not found or already completed: %s", jobID)
+	}
+	return nil
+}
+
+// PurgeHistory removes old job history
+func (m *Manager) PurgeHistory(beforeTime time.Time) int {
+	return m.queue.PurgeOldHistory(beforeTime)
+}
+
+// SaveConfig saves plugin configuration
+func (m *Manager) SaveConfig(config *PluginConfig, backup bool) error {
+	return m.configMgr.SaveConfig(config, backup)
+}
+
+// LoadConfig loads plugin configuration
+func (m *Manager) LoadConfig(pluginID string) (*PluginConfig, error) {
+	config, err := m.configMgr.LoadConfig(pluginID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load config: %w", err)
+	}
+	return config, nil
+}
+
+// ListConfigs returns all loaded configurations
+func (m *Manager) ListConfigs() map[string]*PluginConfig {
+	return m.configMgr.ListConfigs()
+}
+
+// DeleteConfig deletes a configuration
+func (m *Manager) DeleteConfig(pluginID string) error {
+	return m.configMgr.DeleteConfig(pluginID)
+}
+
+// GetRegistry returns the plugin registry
+func (m *Manager) GetRegistry() *Registry {
+	return m.registry
+}
+
+// GetQueue returns the job queue
+func (m *Manager) GetQueue() *JobQueue {
+	return m.queue
+}
+
+// GetDispatcher returns the dispatcher
+func (m *Manager) GetDispatcher() *Dispatcher {
+	return m.dispatcher
+}
+
+// GetGRPCServer returns the gRPC server
+func (m *Manager) GetGRPCServer() *GRPCServer {
+	return m.grpcServer
+}
+
+// GetDetectionHistory returns detection history for a job type
+func (m *Manager) GetDetectionHistory(jobType string) []DetectionRecord {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	configs := m.configMgr.ListConfigs()
+	for _, cfg := range configs {
+		if jobCfg, ok := cfg.GetJobTypeConfig(jobType); ok {
+			cfg.mu.RLock()
+			defer cfg.mu.RUnlock()
+			history := make([]DetectionRecord, len(jobCfg.DetectionHistory))
+			copy(history, jobCfg.DetectionHistory)
+			return history
+		}
+	}
+	return []DetectionRecord{}
+}
+
+// GetExecutionHistory returns execution history for a job type
+func (m *Manager) GetExecutionHistory(jobType string) []ExecutionRecord {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	configs := m.configMgr.ListConfigs()
+	for _, cfg := range configs {
+		if jobCfg, ok := cfg.GetJobTypeConfig(jobType); ok {
+			cfg.mu.RLock()
+			defer cfg.mu.RUnlock()
+			history := make([]ExecutionRecord, len(jobCfg.ExecutionHistory))
+			copy(history, jobCfg.ExecutionHistory)
+			return history
+		}
+	}
+	return []ExecutionRecord{}
+}
+
+// RecordDetection adds a detection record to history
+func (m *Manager) RecordDetection(jobType string, record *DetectionRecord) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	configs := m.configMgr.ListConfigs()
+	for _, cfg := range configs {
+		if jobCfg, ok := cfg.GetJobTypeConfig(jobType); ok {
+			cfg.mu.Lock()
+			maxSize := 50
+			jobCfg.DetectionHistory = append([]DetectionRecord{*record}, jobCfg.DetectionHistory...)
+			if len(jobCfg.DetectionHistory) > maxSize {
+				jobCfg.DetectionHistory = jobCfg.DetectionHistory[:maxSize]
+			}
+			cfg.mu.Unlock()
+			break
+		}
+	}
+}
+
+// RecordExecution adds an execution record to history
+func (m *Manager) RecordExecution(jobType string, record *ExecutionRecord) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	configs := m.configMgr.ListConfigs()
+	for _, cfg := range configs {
+		if jobCfg, ok := cfg.GetJobTypeConfig(jobType); ok {
+			cfg.mu.Lock()
+			maxSize := 100
+			jobCfg.ExecutionHistory = append([]ExecutionRecord{*record}, jobCfg.ExecutionHistory...)
+			if len(jobCfg.ExecutionHistory) > maxSize {
+				jobCfg.ExecutionHistory = jobCfg.ExecutionHistory[:maxSize]
+			}
+			cfg.mu.Unlock()
+			break
+		}
+	}
+}

weed/admin/plugin/registry.go

@@ -0,0 +1,311 @@
+package plugin
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+// Registry manages plugin registration and lifecycle
+type Registry struct {
+	mu                     sync.RWMutex
+	plugins                map[string]*ConnectedPlugin
+	capabilityIndex        map[string][]string // Maps capability to plugin IDs
+	healthCheckTimeout     time.Duration
+	failureDetectionWindow time.Duration
+	failureThreshold       int
+	pluginFailureCount     map[string]int
+}
+
+// NewRegistry creates a new plugin registry
+func NewRegistry(healthCheckTimeout, failureDetectionWindow time.Duration, failureThreshold int) *Registry {
+	return &Registry{
+		plugins:                make(map[string]*ConnectedPlugin),
+		capabilityIndex:        make(map[string][]string),
+		healthCheckTimeout:     healthCheckTimeout,
+		failureDetectionWindow: failureDetectionWindow,
+		failureThreshold:       failureThreshold,
+		pluginFailureCount:     make(map[string]int),
+	}
+}
+
+// RegisterPlugin adds a plugin to the registry
+func (r *Registry) RegisterPlugin(plugin *ConnectedPlugin) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if _, exists := r.plugins[plugin.ID]; exists {
+		return fmt.Errorf("plugin %s already registered", plugin.ID)
+	}
+
+	r.plugins[plugin.ID] = plugin
+	r.pluginFailureCount[plugin.ID] = 0
+
+	// Build capability index
+	for _, cap := range plugin.Capabilities {
+		r.capabilityIndex[cap] = append(r.capabilityIndex[cap], plugin.ID)
+	}
+
+	return nil
+}
+
+// UnregisterPlugin removes a plugin from the registry
+func (r *Registry) UnregisterPlugin(pluginID string) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	plugin, exists := r.plugins[pluginID]
+	if !exists {
+		return fmt.Errorf("plugin %s not found", pluginID)
+	}
+
+	// Remove from capability index
+	for _, cap := range plugin.Capabilities {
+		for i, id := range r.capabilityIndex[cap] {
+			if id == pluginID {
+				r.capabilityIndex[cap] = append(r.capabilityIndex[cap][:i], r.capabilityIndex[cap][i+1:]...)
+				break
+			}
+		}
+	}
+
+	delete(r.plugins, pluginID)
+	delete(r.pluginFailureCount, pluginID)
+
+	return nil
+}
+
+// GetPlugin retrieves a plugin by ID
+func (r *Registry) GetPlugin(pluginID string) (*ConnectedPlugin, error) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	plugin, exists := r.plugins[pluginID]
+	if !exists {
+		return nil, fmt.Errorf("plugin %s not found", pluginID)
+	}
+
+	return plugin, nil
+}
+
+// GetPluginsByCapability returns all plugins with a specific capability
+func (r *Registry) GetPluginsByCapability(capability string) []*ConnectedPlugin {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	pluginIDs, exists := r.capabilityIndex[capability]
+	if !exists {
+		return []*ConnectedPlugin{}
+	}
+
+	var result []*ConnectedPlugin
+	for _, id := range pluginIDs {
+		if plugin, ok := r.plugins[id]; ok {
+			result = append(result, plugin)
+		}
+	}
+
+	return result
+}
+
+// ListPlugins returns all registered plugins
+func (r *Registry) ListPlugins(includeUnhealthy bool) []*ConnectedPlugin {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	var result []*ConnectedPlugin
+
+	for _, plugin := range r.plugins {
+		if !includeUnhealthy && time.Since(plugin.LastHeartbeat) > r.healthCheckTimeout {
+			continue
+		}
+		result = append(result, plugin)
+	}
+
+	return result
+}
+
+// HealthCheck verifies plugin health based on heartbeat status
+func (r *Registry) HealthCheck(pluginID string) (bool, error) {
+	r.mu.RLock()
+	plugin, exists := r.plugins[pluginID]
+	r.mu.RUnlock()
+
+	if !exists {
+		return false, fmt.Errorf("plugin %s not found", pluginID)
+	}
+
+	isHealthy := plugin.IsHealthy(r.healthCheckTimeout)
+
+	if !isHealthy {
+		r.mu.Lock()
+		r.pluginFailureCount[pluginID]++
+		r.mu.Unlock()
+	} else {
+		r.mu.Lock()
+		r.pluginFailureCount[pluginID] = 0
+		r.mu.Unlock()
+	}
+
+	return isHealthy, nil
+}
+
+// GetFailureCount returns the current failure count for a plugin
+func (r *Registry) GetFailureCount(pluginID string) int {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.pluginFailureCount[pluginID]
+}
+
+// HasExceededFailureThreshold checks if a plugin has exceeded the failure threshold
+func (r *Registry) HasExceededFailureThreshold(pluginID string) bool {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.pluginFailureCount[pluginID] > r.failureThreshold
+}
+
+// ResetFailureCount resets the failure counter for a plugin
+func (r *Registry) ResetFailureCount(pluginID string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.pluginFailureCount[pluginID] = 0
+}
+
+// Count returns the total number of registered plugins
+func (r *Registry) Count() int {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return len(r.plugins)
+}
+
+// CountHealthy returns the number of healthy plugins
+func (r *Registry) CountHealthy() int {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	count := 0
+	for _, plugin := range r.plugins {
+		if plugin.IsHealthy(r.healthCheckTimeout) {
+			count++
+		}
+	}
+	return count
+}
+
+// GetCapabilities returns all registered capabilities
+func (r *Registry) GetCapabilities() []string {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	var capabilities []string
+	for cap := range r.capabilityIndex {
+		capabilities = append(capabilities, cap)
+	}
+	return capabilities
+}
+
+// UpdateHeartbeat updates the heartbeat timestamp for a plugin
+func (r *Registry) UpdateHeartbeat(pluginID string) error {
+	r.mu.RLock()
+	plugin, exists := r.plugins[pluginID]
+	r.mu.RUnlock()
+
+	if !exists {
+		return fmt.Errorf("plugin %s not found", pluginID)
+	}
+
+	plugin.UpdateHeartbeat()
+	r.ResetFailureCount(pluginID)
+	return nil
+}
+
+// GetUnhealthyPlugins returns plugins that have failed health checks
+func (r *Registry) GetUnhealthyPlugins() []*ConnectedPlugin {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	var unhealthy []*ConnectedPlugin
+	for _, plugin := range r.plugins {
+		if !plugin.IsHealthy(r.healthCheckTimeout) {
+			unhealthy = append(unhealthy, plugin)
+		}
+	}
+	return unhealthy
+}
+
+// RemoveUnhealthyPlugins removes plugins that have exceeded the failure threshold
+func (r *Registry) RemoveUnhealthyPlugins() []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	var removed []string
+	for pluginID, failureCount := range r.pluginFailureCount {
+		if failureCount > r.failureThreshold {
+			if plugin, exists := r.plugins[pluginID]; exists {
+				// Remove from capability index
+				for _, cap := range plugin.Capabilities {
+					for i, id := range r.capabilityIndex[cap] {
+						if id == pluginID {
+							r.capabilityIndex[cap] = append(r.capabilityIndex[cap][:i], r.capabilityIndex[cap][i+1:]...)
+							break
+						}
+					}
+				}
+				delete(r.plugins, pluginID)
+				delete(r.pluginFailureCount, pluginID)
+				removed = append(removed, pluginID)
+			}
+		}
+	}
+	return removed
+}
+
+// UpdatePluginStatus updates the status field of a plugin
+func (r *Registry) UpdatePluginStatus(pluginID, status string) error {
+	r.mu.RLock()
+	plugin, exists := r.plugins[pluginID]
+	r.mu.RUnlock()
+
+	if !exists {
+		return fmt.Errorf("plugin %s not found", pluginID)
+	}
+
+	plugin.mu.Lock()
+	plugin.Status = status
+	plugin.mu.Unlock()
+
+	return nil
+}
+
+// GetStats returns statistics for all plugins
+func (r *Registry) GetStats() map[string]interface{} {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	totalPlugins := len(r.plugins)
+	healthyPlugins := 0
+	totalActiveJobs := 0
+	totalCompletedJobs := 0
+	totalFailedJobs := 0
+
+	for _, plugin := range r.plugins {
+		if plugin.IsHealthy(r.healthCheckTimeout) {
+			healthyPlugins++
+		}
+		plugin.mu.RLock()
+		totalActiveJobs += plugin.ActiveJobs
+		totalCompletedJobs += plugin.CompletedJobs
+		totalFailedJobs += plugin.FailedJobs
+		plugin.mu.RUnlock()
+	}
+
+	return map[string]interface{}{
+		"total_plugins":     totalPlugins,
+		"healthy_plugins":   healthyPlugins,
+		"unhealthy_plugins": totalPlugins - healthyPlugins,
+		"total_active_jobs": totalActiveJobs,
+		"total_completed":   totalCompletedJobs,
+		"total_failed":      totalFailedJobs,
+		"capabilities":      len(r.capabilityIndex),
+	}
+}

weed/admin/plugin/testing/errors.go

@@ -0,0 +1,6 @@
+package testing
+
+import "errors"
+
+// ErrSimulatedError is returned when error simulation is enabled
+var ErrSimulatedError = errors.New("simulated plugin error")

weed/admin/plugin/testing/harness.go

@@ -0,0 +1,530 @@
+package testing
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+)
+
+// TestHarness provides a complete testing environment for plugins
+type TestHarness struct {
+	mu                sync.RWMutex
+	adminService      *MockPluginService
+	plugins           map[string]*MockPlugin
+	registrations     map[string]*RegistrationRecord
+	jobs              map[string]*JobTracker
+	detections        map[string][]*DetectionRecord
+	executions        map[string]*ExecutionRecord
+	startTime         time.Time
+	timeout           time.Duration
+	testName          string
+	registrationWait  time.Duration
+	executionWait     time.Duration
+	expectedPlugins   int
+	registeredPlugins int
+	failureReasons    []string
+}
+
+// RegistrationRecord tracks plugin registration details
+type RegistrationRecord struct {
+	PluginID           string
+	RegisteredAt       time.Time
+	Version            string
+	Capabilities      []string
+	MaxConcurrentJobs int
+	Status             string
+}
+
+// JobTracker tracks job lifecycle
+type JobTracker struct {
+	JobID        string
+	Type         string
+	PluginID     string
+	Status       plugin_pb.ExecutionStatus
+	CreatedAt    time.Time
+	StartedAt    *time.Time
+	CompletedAt  *time.Time
+	Result       *plugin_pb.JobResult
+	ErrorMessage string
+	Detections   []*DetectionRecord
+}
+
+// DetectionRecord represents a detection result
+type DetectionRecord struct {
+	ResourceID    string
+	DetectionType string
+	Severity      string
+	Description   string
+	Data          []byte
+}
+
+// ExecutionRecord tracks execution details
+type ExecutionRecord struct {
+	ResourceID   string
+	Type         string
+	ExecutedAt   time.Time
+	CompletedAt  *time.Time
+	Success      bool
+	ErrorMessage string
+	Data         []byte
+}
+
+// NewTestHarness creates a new test harness
+func NewTestHarness(testName string) *TestHarness {
+	return &TestHarness{
+		testName:         testName,
+		adminService:     NewMockPluginService(),
+		plugins:          make(map[string]*MockPlugin),
+		registrations:    make(map[string]*RegistrationRecord),
+		jobs:             make(map[string]*JobTracker),
+		detections:       make(map[string][]*DetectionRecord),
+		executions:       make(map[string]*ExecutionRecord),
+		startTime:        time.Now(),
+		timeout:          10 * time.Second,
+		registrationWait: 100 * time.Millisecond,
+		executionWait:    100 * time.Millisecond,
+		failureReasons:   make([]string, 0),
+	}
+}
+
+// SetTimeout sets the overall test timeout
+func (h *TestHarness) SetTimeout(timeout time.Duration) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.timeout = timeout
+}
+
+// SetRegistrationWait sets the wait time for plugin registration
+func (h *TestHarness) SetRegistrationWait(duration time.Duration) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.registrationWait = duration
+}
+
+// SetExecutionWait sets the wait time for job execution
+func (h *TestHarness) SetExecutionWait(duration time.Duration) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.executionWait = duration
+}
+
+// RegisterPlugin simulates plugin registration
+func (h *TestHarness) RegisterPlugin(plugin *MockPlugin) error {
+	h.mu.Lock()
+
+	if plugin == nil {
+		h.failureReasons = append(h.failureReasons, "plugin is nil")
+		h.mu.Unlock()
+		return fmt.Errorf("plugin is nil")
+	}
+
+	h.plugins[plugin.ID] = plugin
+	h.mu.Unlock()
+
+	// Simulate registration with admin service
+	req := &plugin_pb.PluginConnectRequest{
+		PluginId:           plugin.ID,
+		PluginName:         plugin.Name,
+		Version:            plugin.Version,
+		Capabilities:       plugin.Capabilities,
+		CapabilitiesDetail: plugin.CapabilitiesDetail,
+		MaxConcurrentJobs:  int32(plugin.MaxConcurrentJobs),
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
+	defer cancel()
+
+	resp, err := h.adminService.Connect(ctx, req)
+	if err != nil {
+		h.mu.Lock()
+		h.failureReasons = append(h.failureReasons, fmt.Sprintf("registration failed: %v", err))
+		h.mu.Unlock()
+		return err
+	}
+
+	if !resp.Success {
+		h.mu.Lock()
+		h.failureReasons = append(h.failureReasons, "registration response was not successful")
+		h.mu.Unlock()
+		return fmt.Errorf("registration failed: %s", resp.Message)
+	}
+
+	h.mu.Lock()
+	h.registrations[plugin.ID] = &RegistrationRecord{
+		PluginID:           plugin.ID,
+		RegisteredAt:       time.Now(),
+		Version:            plugin.Version,
+		Capabilities:       plugin.Capabilities,
+		MaxConcurrentJobs:  plugin.MaxConcurrentJobs,
+		Status:             "registered",
+	}
+	h.registeredPlugins++
+	h.mu.Unlock()
+
+	return nil
+}
+
+// RegisterMultiplePlugins registers multiple plugins
+func (h *TestHarness) RegisterMultiplePlugins(plugins ...*MockPlugin) error {
+	for _, plugin := range plugins {
+		if err := h.RegisterPlugin(plugin); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ExpectPlugins sets the expected number of plugins
+func (h *TestHarness) ExpectPlugins(count int) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.expectedPlugins = count
+}
+
+// DispatchJob sends a job to a plugin
+func (h *TestHarness) DispatchJob(pluginID string, jobType string, payload *plugin_pb.JobPayload) (string, error) {
+	h.mu.RLock()
+	plugin, ok := h.plugins[pluginID]
+	h.mu.RUnlock()
+
+	if !ok {
+		return "", fmt.Errorf("plugin not found: %s", pluginID)
+	}
+
+	jobID := fmt.Sprintf("job-%d-%d", len(h.jobs), time.Now().UnixNano())
+
+	req := &plugin_pb.ExecuteJobRequest{
+		JobId:   jobID,
+		JobType: jobType,
+		Payload: payload,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
+	defer cancel()
+
+	// Simulate job dispatch
+	err := h.adminService.SimulateJobExecution(req)
+	if err != nil {
+		h.mu.Lock()
+		h.failureReasons = append(h.failureReasons, fmt.Sprintf("job dispatch failed: %v", err))
+		h.mu.Unlock()
+		return "", err
+	}
+
+	// Wait for job to complete
+	time.Sleep(h.executionWait)
+
+	// Verify job execution
+	plugin.TrackJob(req)
+
+	_, executionErr := plugin.ExecuteJob(ctx, jobID, jobType, payload)
+
+	h.mu.Lock()
+	h.jobs[jobID] = &JobTracker{
+		JobID:      jobID,
+		Type:       jobType,
+		PluginID:   pluginID,
+		Status:     plugin_pb.ExecutionStatus_EXECUTION_STATUS_RUNNING,
+		CreatedAt:  time.Now(),
+	}
+	h.mu.Unlock()
+
+	// Simulate completion after a small delay
+	time.Sleep(h.executionWait)
+
+	h.mu.Lock()
+	h.jobs[jobID].Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED
+	now := time.Now()
+	h.jobs[jobID].CompletedAt = &now
+	h.mu.Unlock()
+
+	if executionErr != nil {
+		h.mu.Lock()
+		h.jobs[jobID].Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_FAILED
+		h.jobs[jobID].ErrorMessage = executionErr.Error()
+		h.mu.Unlock()
+	}
+
+	return jobID, nil
+}
+
+// VerifyRegistration checks if a plugin was registered
+func (h *TestHarness) VerifyRegistration(pluginID string) bool {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+	_, ok := h.registrations[pluginID]
+	return ok
+}
+
+// VerifyJobCompleted checks if a job completed successfully
+func (h *TestHarness) VerifyJobCompleted(jobID string) bool {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	job, ok := h.jobs[jobID]
+	if !ok {
+		return false
+	}
+
+	return job.Status == plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED
+}
+
+// VerifyJobFailed checks if a job failed
+func (h *TestHarness) VerifyJobFailed(jobID string) bool {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	job, ok := h.jobs[jobID]
+	if !ok {
+		return false
+	}
+
+	return job.Status == plugin_pb.ExecutionStatus_EXECUTION_STATUS_FAILED
+}
+
+// VerifyPluginCapability checks if a plugin has a capability
+func (h *TestHarness) VerifyPluginCapability(pluginID string, capability string) bool {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	reg, ok := h.registrations[pluginID]
+	if !ok {
+		return false
+	}
+
+	for _, cap := range reg.Capabilities {
+		if cap == capability {
+			return true
+		}
+	}
+
+	return false
+}
+
+// GetJobStatus returns the status of a job
+func (h *TestHarness) GetJobStatus(jobID string) plugin_pb.ExecutionStatus {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	job, ok := h.jobs[jobID]
+	if !ok {
+		return plugin_pb.ExecutionStatus_EXECUTION_STATUS_UNKNOWN
+	}
+
+	return job.Status
+}
+
+// GetPlugin returns a registered plugin
+func (h *TestHarness) GetPlugin(pluginID string) *MockPlugin {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+	return h.plugins[pluginID]
+}
+
+// GetRegistrationCount returns the number of registered plugins
+func (h *TestHarness) GetRegistrationCount() int {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+	return h.registeredPlugins
+}
+
+// GetJobCount returns the total number of jobs dispatched
+func (h *TestHarness) GetJobCount() int {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+	return len(h.jobs)
+}
+
+// SimulateDetection simulates detection results
+func (h *TestHarness) SimulateDetection(pluginID string, result *DetectionRecord) error {
+	h.mu.RLock()
+	plugin, ok := h.plugins[pluginID]
+	h.mu.RUnlock()
+
+	if !ok {
+		return fmt.Errorf("plugin not found: %s", pluginID)
+	}
+
+	plugin.AddDetectionResult(result.ResourceID, result.DetectionType, result.Severity, result.Description, result.Data)
+
+	h.mu.Lock()
+	if _, exists := h.detections[pluginID]; !exists {
+		h.detections[pluginID] = make([]*DetectionRecord, 0)
+	}
+	h.detections[pluginID] = append(h.detections[pluginID], result)
+	h.mu.Unlock()
+
+	return nil
+}
+
+// GetAdminService returns the underlying admin service
+func (h *TestHarness) GetAdminService() *MockPluginService {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+	return h.adminService
+}
+
+// GetTestDuration returns the elapsed test time
+func (h *TestHarness) GetTestDuration() time.Duration {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+	return time.Since(h.startTime)
+}
+
+// ReportFailure records a test failure reason
+func (h *TestHarness) ReportFailure(reason string) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.failureReasons = append(h.failureReasons, reason)
+}
+
+// HasFailures checks if any failures were recorded
+func (h *TestHarness) HasFailures() bool {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+	return len(h.failureReasons) > 0
+}
+
+// GetFailures returns all recorded failures
+func (h *TestHarness) GetFailures() []string {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+	failures := make([]string, len(h.failureReasons))
+	copy(failures, h.failureReasons)
+	return failures
+}
+
+// WaitForRegistration waits for a specific number of plugins to register
+func (h *TestHarness) WaitForRegistration(count int, timeout time.Duration) bool {
+	deadline := time.Now().Add(timeout)
+	for {
+		h.mu.RLock()
+		current := h.registeredPlugins
+		h.mu.RUnlock()
+
+		if current >= count {
+			return true
+		}
+
+		if time.Now().After(deadline) {
+			return false
+		}
+
+		time.Sleep(10 * time.Millisecond)
+	}
+}
+
+// VerifyAdminServiceStats checks admin service statistics
+func (h *TestHarness) VerifyAdminServiceStats(regCount, jobCount int) bool {
+	return h.adminService.GetRegistrationCount() == regCount &&
+		h.adminService.GetJobDispatchCount() == jobCount
+}
+
+// GetCompletedJobCount returns the number of completed jobs
+func (h *TestHarness) GetCompletedJobCount() int {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	count := 0
+	for _, job := range h.jobs {
+		if job.Status == plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED {
+			count++
+		}
+	}
+	return count
+}
+
+// GetFailedJobCount returns the number of failed jobs
+func (h *TestHarness) GetFailedJobCount() int {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	count := 0
+	for _, job := range h.jobs {
+		if job.Status == plugin_pb.ExecutionStatus_EXECUTION_STATUS_FAILED {
+			count++
+		}
+	}
+	return count
+}
+
+// Cleanup performs cleanup after a test
+func (h *TestHarness) Cleanup() {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	// Reset all plugins
+	for _, plugin := range h.plugins {
+		plugin.Reset()
+	}
+
+	// Reset admin service
+	h.adminService.ResetCounters()
+
+	// Clear tracking
+	h.registrations = make(map[string]*RegistrationRecord)
+	h.jobs = make(map[string]*JobTracker)
+	h.detections = make(map[string][]*DetectionRecord)
+	h.executions = make(map[string]*ExecutionRecord)
+	h.failureReasons = make([]string, 0)
+	h.registeredPlugins = 0
+	h.startTime = time.Now()
+}
+
+// MockExecuteJobStream is a mock implementation of the ExecuteJob stream
+type MockExecuteJobStream struct {
+	responses []*plugin_pb.ExecuteJobResponse
+	mu        sync.Mutex
+}
+
+// Send sends a response on the stream
+func (m *MockExecuteJobStream) Send(resp *plugin_pb.ExecuteJobResponse) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.responses = append(m.responses, resp)
+	return nil
+}
+
+// Recv receives a response from the stream
+func (m *MockExecuteJobStream) Recv() (*plugin_pb.ExecuteJobResponse, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if len(m.responses) == 0 {
+		return nil, fmt.Errorf("no responses")
+	}
+	resp := m.responses[0]
+	m.responses = m.responses[1:]
+	return resp, nil
+}
+
+// SetHeader sets the metadata header
+func (m *MockExecuteJobStream) SetHeader(map[string][]string) error {
+	return nil
+}
+
+// SendHeader sends the metadata header
+func (m *MockExecuteJobStream) SendHeader(map[string][]string) error {
+	return nil
+}
+
+// SetTrailer sets the metadata trailer
+func (m *MockExecuteJobStream) SetTrailer(map[string][]string) {
+}
+
+// Context returns the context
+func (m *MockExecuteJobStream) Context() context.Context {
+	return context.Background()
+}
+
+// SendMsg sends a message on the stream
+func (m *MockExecuteJobStream) SendMsg(interface{}) error {
+	return nil
+}
+
+// RecvMsg receives a message from the stream
+func (m *MockExecuteJobStream) RecvMsg(interface{}) error {
+	return nil
+}

weed/admin/plugin/testing/mock_admin.go

@@ -0,0 +1,342 @@
+package testing
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+)
+
+// MockPluginService simulates the admin-side PluginService for testing
+type MockPluginService struct {
+	mu                   sync.RWMutex
+	plugins              map[string]*MockPluginInstance
+	jobs                 map[string]*MockJob
+	jobCounter           int
+	heartbeats           map[string]*plugin_pb.HealthReport
+	lastHeartbeatTime    map[string]time.Time
+	jobDispatchCalls     int
+	registrationCalls    int
+	receivedHealthReports []plugin_pb.HealthReport
+}
+
+// MockPluginInstance tracks a registered plugin
+type MockPluginInstance struct {
+	ID                    string
+	Name                  string
+	Version               string
+	Status                string
+	Capabilities         []string
+	MaxConcurrentJobs    int
+	ConnectedAt          time.Time
+	LastHeartbeat        time.Time
+	ActiveJobCount       int
+	CompletedJobCount    int
+	FailedJobCount       int
+	CapabilitiesDetail   *plugin_pb.PluginCapabilities
+	Metadata             map[string]string
+}
+
+// MockJob represents a job dispatched to a plugin
+type MockJob struct {
+	ID            string
+	Type          string
+	PluginID      string
+	Payload       *plugin_pb.JobPayload
+	Timeout       time.Duration
+	RetryCount    int
+	Context       map[string]string
+	Status        plugin_pb.ExecutionStatus
+	DispatchedAt  time.Time
+	ExecutedAt    *time.Time
+	Result        *plugin_pb.JobResult
+	ResultMessage string
+	StreamCalls   int
+}
+
+// NewMockPluginService creates a new mock admin service
+func NewMockPluginService() *MockPluginService {
+	return &MockPluginService{
+		plugins:               make(map[string]*MockPluginInstance),
+		jobs:                  make(map[string]*MockJob),
+		heartbeats:            make(map[string]*plugin_pb.HealthReport),
+		lastHeartbeatTime:     make(map[string]time.Time),
+		receivedHealthReports: make([]plugin_pb.HealthReport, 0),
+	}
+}
+
+// Connect handles plugin registration
+func (m *MockPluginService) Connect(ctx context.Context, req *plugin_pb.PluginConnectRequest) (*plugin_pb.PluginConnectResponse, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	m.registrationCalls++
+
+	// Register the plugin
+	plugin := &MockPluginInstance{
+		ID:                  req.PluginId,
+		Name:                req.PluginName,
+		Version:             req.Version,
+		Status:              "connected",
+		Capabilities:        req.Capabilities,
+		MaxConcurrentJobs:   int(req.MaxConcurrentJobs),
+		ConnectedAt:         time.Now(),
+		LastHeartbeat:       time.Now(),
+		CapabilitiesDetail:  req.CapabilitiesDetail,
+		Metadata:            req.Metadata,
+	}
+	m.plugins[req.PluginId] = plugin
+	m.lastHeartbeatTime[req.PluginId] = time.Now()
+
+	// Build response with assigned types
+	assignedTypes := req.Capabilities
+
+	config := &plugin_pb.PluginConfig{
+		PluginId:   req.PluginId,
+		Properties: make(map[string]string),
+		JobTypes:   make([]*plugin_pb.JobTypeConfig, 0),
+	}
+
+	return &plugin_pb.PluginConnectResponse{
+		Success:      true,
+		Message:      "Plugin registered successfully",
+		MasterId:     "mock-master-001",
+		AssignedTypes: assignedTypes,
+		Config:       config,
+	}, nil
+}
+
+// SimulateJobExecution simulates job execution
+func (m *MockPluginService) SimulateJobExecution(req *plugin_pb.ExecuteJobRequest) error {
+	m.mu.Lock()
+
+	m.jobDispatchCalls++
+
+	// Create job entry
+	job := &MockJob{
+		ID:           req.JobId,
+		Type:         req.JobType,
+		Payload:      req.Payload,
+		Timeout:      durationFromProto(req.Timeout),
+		RetryCount:   int(req.RetryCount),
+		Context:      req.Context,
+		DispatchedAt: time.Now(),
+		Status:       plugin_pb.ExecutionStatus_EXECUTION_STATUS_ACCEPTED,
+		StreamCalls:  0,
+	}
+	m.jobs[req.JobId] = job
+	m.mu.Unlock()
+
+	// Simulate job execution
+	time.Sleep(50 * time.Millisecond)
+
+	// Update job status
+	m.mu.Lock()
+	job.StreamCalls++
+	job.Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_RUNNING
+	m.mu.Unlock()
+
+	// Simulate processing
+	time.Sleep(50 * time.Millisecond)
+
+	m.mu.Lock()
+	job.StreamCalls++
+	job.Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED
+	job.ResultMessage = "Job completed successfully"
+	now := time.Now()
+	job.ExecutedAt = &now
+	m.mu.Unlock()
+
+	return nil
+}
+
+// ExecuteJob simulates job dispatch
+func (m *MockPluginService) ExecuteJob(ctx context.Context, req *plugin_pb.ExecuteJobRequest) (*plugin_pb.ExecuteJobResponse, error) {
+	m.mu.Lock()
+	m.jobDispatchCalls++
+	m.mu.Unlock()
+
+	return &plugin_pb.ExecuteJobResponse{
+		JobId:   req.JobId,
+		Status:  plugin_pb.ExecutionStatus_EXECUTION_STATUS_ACCEPTED,
+		Message: "Job accepted",
+	}, nil
+}
+
+// ReportHealth handles plugin health reports
+func (m *MockPluginService) ReportHealth(ctx context.Context, report *plugin_pb.HealthReport) (*plugin_pb.HealthReportResponse, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	m.heartbeats[report.PluginId] = report
+	m.lastHeartbeatTime[report.PluginId] = time.Now()
+	m.receivedHealthReports = append(m.receivedHealthReports, *report)
+
+	// Update plugin status
+	if plugin, ok := m.plugins[report.PluginId]; ok {
+		plugin.LastHeartbeat = time.Now()
+		plugin.ActiveJobCount = int(report.ActiveJobs)
+	}
+
+	return &plugin_pb.HealthReportResponse{
+		Acknowledged: true,
+		Feedback:     "Health report received",
+	}, nil
+}
+
+// GetConfig handles config retrieval
+func (m *MockPluginService) GetConfig(ctx context.Context, req *plugin_pb.GetConfigRequest) (*plugin_pb.GetConfigResponse, error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	config := &plugin_pb.PluginConfig{
+		PluginId:   req.PluginId,
+		Properties: make(map[string]string),
+		JobTypes:   make([]*plugin_pb.JobTypeConfig, 0),
+	}
+
+	return &plugin_pb.GetConfigResponse{
+		Config:  config,
+		Version: 1,
+	}, nil
+}
+
+// SubmitResult handles job result submission
+func (m *MockPluginService) SubmitResult(ctx context.Context, req *plugin_pb.JobResultRequest) (*plugin_pb.JobResultResponse, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if job, ok := m.jobs[req.JobId]; ok {
+		job.Status = req.Status
+		job.Result = req.Result
+		job.ResultMessage = req.Message
+	}
+
+	return &plugin_pb.JobResultResponse{
+		Acknowledged:   true,
+		ActionsToTake: []string{},
+	}, nil
+}
+
+// GetRegistrationCount returns how many times Connect was called
+func (m *MockPluginService) GetRegistrationCount() int {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.registrationCalls
+}
+
+// GetJobDispatchCount returns how many times ExecuteJob was called
+func (m *MockPluginService) GetJobDispatchCount() int {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.jobDispatchCalls
+}
+
+// GetPluginCount returns the number of registered plugins
+func (m *MockPluginService) GetPluginCount() int {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return len(m.plugins)
+}
+
+// GetPlugin returns a registered plugin by ID
+func (m *MockPluginService) GetPlugin(pluginID string) *MockPluginInstance {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.plugins[pluginID]
+}
+
+// GetJob returns a dispatched job by ID
+func (m *MockPluginService) GetJob(jobID string) *MockJob {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.jobs[jobID]
+}
+
+// GetJobCount returns the total number of dispatched jobs
+func (m *MockPluginService) GetJobCount() int {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return len(m.jobs)
+}
+
+// GetLastHeartbeat returns the last heartbeat time for a plugin
+func (m *MockPluginService) GetLastHeartbeat(pluginID string) *time.Time {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	if t, ok := m.lastHeartbeatTime[pluginID]; ok {
+		return &t
+	}
+	return nil
+}
+
+// GetHeartbeatCount returns how many heartbeats have been received
+func (m *MockPluginService) GetHeartbeatCount() int {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return len(m.receivedHealthReports)
+}
+
+// ResetCounters resets all counters for a fresh test
+func (m *MockPluginService) ResetCounters() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.registrationCalls = 0
+	m.jobDispatchCalls = 0
+	m.plugins = make(map[string]*MockPluginInstance)
+	m.jobs = make(map[string]*MockJob)
+	m.heartbeats = make(map[string]*plugin_pb.HealthReport)
+	m.lastHeartbeatTime = make(map[string]time.Time)
+	m.receivedHealthReports = make([]plugin_pb.HealthReport, 0)
+}
+
+// VerifyJobCompleted checks if a job was completed successfully
+func (m *MockPluginService) VerifyJobCompleted(jobID string) bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	job, ok := m.jobs[jobID]
+	if !ok {
+		return false
+	}
+	return job.Status == plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED
+}
+
+// VerifyJobFailed checks if a job failed
+func (m *MockPluginService) VerifyJobFailed(jobID string) bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	job, ok := m.jobs[jobID]
+	if !ok {
+		return false
+	}
+	return job.Status == plugin_pb.ExecutionStatus_EXECUTION_STATUS_FAILED
+}
+
+// GetJobStatus returns the current status of a job
+func (m *MockPluginService) GetJobStatus(jobID string) plugin_pb.ExecutionStatus {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	if job, ok := m.jobs[jobID]; ok {
+		return job.Status
+	}
+	return plugin_pb.ExecutionStatus_EXECUTION_STATUS_UNKNOWN
+}
+
+// VerifyPluginRegistered checks if a plugin is registered
+func (m *MockPluginService) VerifyPluginRegistered(pluginID string) bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	_, ok := m.plugins[pluginID]
+	return ok
+}
+
+// durationFromProto converts proto Duration to time.Duration
+func durationFromProto(d *durationpb.Duration) time.Duration {
+	if d == nil {
+		return 0
+	}
+	return time.Duration(d.Seconds)*time.Second + time.Duration(d.Nanos)
+}

weed/admin/plugin/testing/mock_plugin.go

@@ -0,0 +1,449 @@
+package testing
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+)
+
+// MockPlugin simulates a plugin worker instance for testing
+type MockPlugin struct {
+	mu                      sync.RWMutex
+	ID                      string
+	Name                    string
+	Version                 string
+	Status                  string
+	Capabilities            []string
+	CapabilitiesDetail      *plugin_pb.PluginCapabilities
+	MaxConcurrentJobs       int
+	Config                  *plugin_pb.PluginConfig
+	ActiveJobs              map[string]*MockJobExecution
+	CompletedJobs           int
+	FailedJobs              int
+	ConnectStreamCalls      int
+	ExecuteJobStreamCalls   int
+	ReportHealthCalls       int
+	GetConfigCalls          int
+	SubmitResultCalls       int
+	ReceivedJobs            []*plugin_pb.ExecuteJobRequest
+	ReceivedHealthReports   []*plugin_pb.HealthReport
+	LastError               string
+	SimulateError           bool
+	SimulateErrorType       string
+	SchemaData              []byte
+	DetectionResults        []*DetectionResult
+	ExecutionResults        []*ExecutionResult
+}
+
+// MockJobExecution tracks job execution state
+type MockJobExecution struct {
+	JobID        string
+	Type         string
+	Status       plugin_pb.ExecutionStatus
+	Progress     float32
+	CurrentStep  string
+	StartTime    time.Time
+	EndTime      *time.Time
+	Result       *plugin_pb.JobResult
+	ErrorMessage string
+}
+
+// DetectionResult represents detection results
+type DetectionResult struct {
+	ResourceID   string
+	DetectionType string
+	Severity     string
+	Description  string
+	Data         []byte
+}
+
+// ExecutionResult represents execution results
+type ExecutionResult struct {
+	ResourceID   string
+	Success      bool
+	ErrorMessage string
+	Data         []byte
+}
+
+// NewMockPlugin creates a new mock plugin
+func NewMockPlugin(id, name, version string) *MockPlugin {
+	return &MockPlugin{
+		ID:                    id,
+		Name:                  name,
+		Version:               version,
+		Status:                "ready",
+		Capabilities:          make([]string, 0),
+		CapabilitiesDetail:    &plugin_pb.PluginCapabilities{},
+		MaxConcurrentJobs:     5,
+		Config:                &plugin_pb.PluginConfig{},
+		ActiveJobs:            make(map[string]*MockJobExecution),
+		ReceivedJobs:          make([]*plugin_pb.ExecuteJobRequest, 0),
+		ReceivedHealthReports: make([]*plugin_pb.HealthReport, 0),
+		DetectionResults:      make([]*DetectionResult, 0),
+		ExecutionResults:      make([]*ExecutionResult, 0),
+	}
+}
+
+// AddCapability adds a capability to the plugin
+func (m *MockPlugin) AddCapability(cap string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.Capabilities = append(m.Capabilities, cap)
+}
+
+// AddDetectionCapability adds a detection capability
+func (m *MockPlugin) AddDetectionCapability(typ, desc string, minInterval int32, requiresFullScan bool) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.CapabilitiesDetail == nil {
+		m.CapabilitiesDetail = &plugin_pb.PluginCapabilities{}
+	}
+	m.CapabilitiesDetail.Detection = append(m.CapabilitiesDetail.Detection, &plugin_pb.DetectionCapability{
+		Type:               typ,
+		Description:        desc,
+		MinIntervalSeconds: minInterval,
+		RequiresFullScan:   requiresFullScan,
+	})
+	m.Capabilities = append(m.Capabilities, typ)
+}
+
+// AddMaintenanceCapability adds a maintenance capability
+func (m *MockPlugin) AddMaintenanceCapability(typ, desc string, requiredDetections []string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.CapabilitiesDetail == nil {
+		m.CapabilitiesDetail = &plugin_pb.PluginCapabilities{}
+	}
+	m.CapabilitiesDetail.Maintenance = append(m.CapabilitiesDetail.Maintenance, &plugin_pb.MaintenanceCapability{
+		Type:                   typ,
+		Description:            desc,
+		RequiredDetectionTypes: requiredDetections,
+	})
+}
+
+// SetSchema sets the schema data
+func (m *MockPlugin) SetSchema(data []byte) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.SchemaData = data
+}
+
+// AddDetectionResult adds a detection result
+func (m *MockPlugin) AddDetectionResult(resourceID, detectionType, severity, description string, data []byte) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.DetectionResults = append(m.DetectionResults, &DetectionResult{
+		ResourceID:    resourceID,
+		DetectionType: detectionType,
+		Severity:      severity,
+		Description:   description,
+		Data:          data,
+	})
+}
+
+// AddExecutionResult adds an execution result
+func (m *MockPlugin) AddExecutionResult(resourceID string, success bool, errorMsg string, data []byte) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.ExecutionResults = append(m.ExecutionResults, &ExecutionResult{
+		ResourceID:   resourceID,
+		Success:      success,
+		ErrorMessage: errorMsg,
+		Data:         data,
+	})
+}
+
+// GetConfigurationSchema implements schema retrieval
+func (m *MockPlugin) GetConfigurationSchema(ctx context.Context) ([]byte, error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	if m.SimulateError && m.SimulateErrorType == "schema" {
+		return nil, ErrSimulatedError
+	}
+
+	return m.SchemaData, nil
+}
+
+// DetectJobs implements detection logic
+func (m *MockPlugin) DetectJobs(ctx context.Context) ([]*DetectionResult, error) {
+	m.mu.Lock()
+	m.ReportHealthCalls++
+	results := make([]*DetectionResult, len(m.DetectionResults))
+	copy(results, m.DetectionResults)
+	m.mu.Unlock()
+
+	if m.SimulateError && m.SimulateErrorType == "detect" {
+		return nil, ErrSimulatedError
+	}
+
+	return results, nil
+}
+
+// ExecuteJob implements job execution
+func (m *MockPlugin) ExecuteJob(ctx context.Context, jobID string, jobType string, payload *plugin_pb.JobPayload) (*ExecutionResult, error) {
+	m.mu.Lock()
+	m.ExecuteJobStreamCalls++
+
+	execution := &MockJobExecution{
+		JobID:      jobID,
+		Type:       jobType,
+		Status:     plugin_pb.ExecutionStatus_EXECUTION_STATUS_RUNNING,
+		StartTime:  time.Now(),
+		Progress:   0,
+		CurrentStep: "initialized",
+	}
+	m.ActiveJobs[jobID] = execution
+	m.mu.Unlock()
+
+	// Simulate execution steps
+	steps := []string{"initialized", "validating", "processing", "finalizing"}
+
+	for i, step := range steps {
+		select {
+		case <-ctx.Done():
+			m.mu.Lock()
+			execution.Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_CANCELLED
+			execution.ErrorMessage = "context cancelled"
+			delete(m.ActiveJobs, jobID)
+			m.mu.Unlock()
+			return nil, ctx.Err()
+		default:
+		}
+
+		m.mu.Lock()
+		execution.CurrentStep = step
+		execution.Progress = float32((i + 1) * 25)
+		m.mu.Unlock()
+
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if m.SimulateError && m.SimulateErrorType == "execute" {
+		execution.Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_FAILED
+		execution.ErrorMessage = "simulated execution error"
+		m.FailedJobs++
+		delete(m.ActiveJobs, jobID)
+		return nil, ErrSimulatedError
+	}
+
+	// Get results
+	result := &ExecutionResult{
+		ResourceID:   jobID,
+		Success:      true,
+		ErrorMessage: "",
+	}
+
+	if len(m.ExecutionResults) > 0 {
+		result = m.ExecutionResults[0]
+		m.ExecutionResults = m.ExecutionResults[1:]
+	}
+
+	execution.Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED
+	execution.Progress = 100
+	execution.CurrentStep = "completed"
+	now := time.Now()
+	execution.EndTime = &now
+
+	m.CompletedJobs++
+	delete(m.ActiveJobs, jobID)
+
+	return result, nil
+}
+
+// ConnectStream simulates the Connect RPC stream
+func (m *MockPlugin) ConnectStream(ctx context.Context, conn grpc.ClientConnInterface) error {
+	m.mu.Lock()
+	m.ConnectStreamCalls++
+	m.mu.Unlock()
+
+	if m.SimulateError && m.SimulateErrorType == "connect" {
+		return ErrSimulatedError
+	}
+
+	return nil
+}
+
+// ExecuteJobStream simulates the ExecuteJob RPC stream
+func (m *MockPlugin) ExecuteJobStream(ctx context.Context, conn grpc.ClientConnInterface, jobID string) error {
+	m.mu.Lock()
+	m.ExecuteJobStreamCalls++
+	m.mu.Unlock()
+
+	if m.SimulateError && m.SimulateErrorType == "executestream" {
+		return ErrSimulatedError
+	}
+
+	return nil
+}
+
+// ReportHealth sends a health report
+func (m *MockPlugin) ReportHealth(ctx context.Context, conn grpc.ClientConnInterface) error {
+	m.mu.Lock()
+	m.ReportHealthCalls++
+
+	activeCount := len(m.ActiveJobs)
+	m.mu.Unlock()
+
+	if m.SimulateError && m.SimulateErrorType == "health" {
+		return ErrSimulatedError
+	}
+
+	report := &plugin_pb.HealthReport{
+		PluginId:   m.ID,
+		TimestampMs: time.Now().UnixMilli(),
+		Status:     plugin_pb.HealthStatus_HEALTH_STATUS_HEALTHY,
+		ActiveJobs: int32(activeCount),
+	}
+
+	m.mu.Lock()
+	m.ReceivedHealthReports = append(m.ReceivedHealthReports, report)
+	m.mu.Unlock()
+
+	return nil
+}
+
+// GetConfig retrieves configuration
+func (m *MockPlugin) GetConfig(ctx context.Context, conn grpc.ClientConnInterface) (*plugin_pb.PluginConfig, error) {
+	m.mu.Lock()
+	m.GetConfigCalls++
+	defer m.mu.Unlock()
+
+	if m.SimulateError && m.SimulateErrorType == "getconfig" {
+		return nil, ErrSimulatedError
+	}
+
+	return m.Config, nil
+}
+
+// SubmitResult submits job results
+func (m *MockPlugin) SubmitResult(ctx context.Context, conn grpc.ClientConnInterface, jobID string, result *plugin_pb.JobResult) error {
+	m.mu.Lock()
+	m.SubmitResultCalls++
+	defer m.mu.Unlock()
+
+	if m.SimulateError && m.SimulateErrorType == "submitresult" {
+		return ErrSimulatedError
+	}
+
+	return nil
+}
+
+// GetActiveJobCount returns the number of active jobs
+func (m *MockPlugin) GetActiveJobCount() int {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return len(m.ActiveJobs)
+}
+
+// GetCompletedJobCount returns the number of completed jobs
+func (m *MockPlugin) GetCompletedJobCount() int {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.CompletedJobs
+}
+
+// GetFailedJobCount returns the number of failed jobs
+func (m *MockPlugin) GetFailedJobCount() int {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.FailedJobs
+}
+
+// GetStreamCallCount returns the count of stream calls
+func (m *MockPlugin) GetStreamCallCount() int {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.ExecuteJobStreamCalls
+}
+
+// GetHealthReportCount returns the count of health reports sent
+func (m *MockPlugin) GetHealthReportCount() int {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.ReportHealthCalls
+}
+
+// EnableErrorSimulation enables error simulation
+func (m *MockPlugin) EnableErrorSimulation(errorType string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.SimulateError = true
+	m.SimulateErrorType = errorType
+}
+
+// DisableErrorSimulation disables error simulation
+func (m *MockPlugin) DisableErrorSimulation() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.SimulateError = false
+	m.SimulateErrorType = ""
+}
+
+// Reset clears all counters and state
+func (m *MockPlugin) Reset() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.ActiveJobs = make(map[string]*MockJobExecution)
+	m.CompletedJobs = 0
+	m.FailedJobs = 0
+	m.ConnectStreamCalls = 0
+	m.ExecuteJobStreamCalls = 0
+	m.ReportHealthCalls = 0
+	m.GetConfigCalls = 0
+	m.SubmitResultCalls = 0
+	m.ReceivedJobs = make([]*plugin_pb.ExecuteJobRequest, 0)
+	m.ReceivedHealthReports = make([]*plugin_pb.HealthReport, 0)
+	m.LastError = ""
+	m.SimulateError = false
+	m.SimulateErrorType = ""
+}
+
+// GetJobExecution returns execution details for a job
+func (m *MockPlugin) GetJobExecution(jobID string) *MockJobExecution {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.ActiveJobs[jobID]
+}
+
+// TrackJob records a received job
+func (m *MockPlugin) TrackJob(req *plugin_pb.ExecuteJobRequest) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.ReceivedJobs = append(m.ReceivedJobs, req)
+}
+
+// GetReceivedJobCount returns the count of received jobs
+func (m *MockPlugin) GetReceivedJobCount() int {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return len(m.ReceivedJobs)
+}
+
+// SimulateStreamError simulates an error during streaming
+func (m *MockPlugin) SimulateStreamError(reason error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.LastError = reason.Error()
+}
+
+// SetStatus sets the plugin status
+func (m *MockPlugin) SetStatus(status string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.Status = status
+}
+
+// GetStatus returns the plugin status
+func (m *MockPlugin) GetStatus() string {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.Status
+}

weed/admin/plugin/types.go

@@ -0,0 +1,265 @@
+package plugin
+
+import (
+	"sync"
+	"time"
+)
+
+// JobState represents the current state of a job in the state machine
+type JobState int
+
+const (
+	JobStatePending JobState = iota
+	JobStateScheduled
+	JobStateRunning
+	JobStateCompleted
+	JobStateFailed
+	JobStateCancelled
+)
+
+func (s JobState) String() string {
+	switch s {
+	case JobStatePending:
+		return "PENDING"
+	case JobStateScheduled:
+		return "SCHEDULED"
+	case JobStateRunning:
+		return "RUNNING"
+	case JobStateCompleted:
+		return "COMPLETED"
+	case JobStateFailed:
+		return "FAILED"
+	case JobStateCancelled:
+		return "CANCELLED"
+	default:
+		return "UNKNOWN"
+	}
+}
+
+// Job represents a detection or maintenance task
+type Job struct {
+	mu               sync.RWMutex
+	ID               string
+	Type             string
+	PluginID         string
+	State            JobState
+	Payload          interface{}
+	CreatedAt        time.Time
+	StartedAt        *time.Time
+	CompletedAt      *time.Time
+	ExecutionTime    time.Duration
+	RetryCount       int
+	MaxRetries       int
+	LastError        string
+	Result           *JobResult
+	DetectionRecords []DetectionRecord
+}
+
+// GetState safely retrieves the job state
+func (j *Job) GetState() JobState {
+	j.mu.RLock()
+	defer j.mu.RUnlock()
+	return j.State
+}
+
+// SetState safely updates the job state
+func (j *Job) SetState(state JobState) {
+	j.mu.Lock()
+	defer j.mu.Unlock()
+	j.State = state
+	if state == JobStateRunning {
+		now := time.Now()
+		j.StartedAt = &now
+	} else if state == JobStateCompleted || state == JobStateFailed || state == JobStateCancelled {
+		now := time.Now()
+		j.CompletedAt = &now
+		if j.StartedAt != nil {
+			j.ExecutionTime = j.CompletedAt.Sub(*j.StartedAt)
+		}
+	}
+}
+
+// JobResult contains the output from job execution
+type JobResult struct {
+	Success   bool
+	Data      []byte
+	Warnings  []string
+	Errors    []string
+	Metadata  map[string]string
+	Message   string
+}
+
+// DetectionRecord represents a single detection result
+type DetectionRecord struct {
+	DetectionType    string
+	Timestamp        time.Time
+	Severity         string
+	Description      string
+	AffectedResource string
+	RawData          []byte
+}
+
+// ExecutionRecord persists job execution history
+type ExecutionRecord struct {
+	JobID       string
+	JobType     string
+	PluginID    string
+	State       JobState
+	CreatedAt   time.Time
+	StartedAt   *time.Time
+	CompletedAt *time.Time
+	Payload     interface{}
+	Result      *JobResult
+	RetryCount  int
+	LastError   string
+}
+
+// ConnectedPlugin represents a connected plugin instance
+type ConnectedPlugin struct {
+	mu                    sync.RWMutex
+	ID                    string
+	Name                  string
+	Version               string
+	Status                string
+	Capabilities         []string
+	MaxConcurrentJobs    int
+	ActiveJobs           int
+	CompletedJobs        int
+	FailedJobs           int
+	TotalDetections      int64
+	AvgExecutionTimeMs   float64
+	CPUUsagePercent      float64
+	MemoryUsageBytes     int64
+	ConnectedAt          time.Time
+	LastHeartbeat        time.Time
+	Metadata             map[string]string
+	HealthCheckInterval  time.Duration
+	JobTimeout           time.Duration
+}
+
+// IsHealthy checks if the plugin is considered healthy based on heartbeat
+func (cp *ConnectedPlugin) IsHealthy(timeout time.Duration) bool {
+	cp.mu.RLock()
+	defer cp.mu.RUnlock()
+	return time.Since(cp.LastHeartbeat) < timeout
+}
+
+// UpdateHeartbeat updates the last heartbeat timestamp
+func (cp *ConnectedPlugin) UpdateHeartbeat() {
+	cp.mu.Lock()
+	defer cp.mu.Unlock()
+	cp.LastHeartbeat = time.Now()
+}
+
+// IncActiveJobs increments active job counter
+func (cp *ConnectedPlugin) IncActiveJobs() {
+	cp.mu.Lock()
+	defer cp.mu.Unlock()
+	cp.ActiveJobs++
+}
+
+// DecActiveJobs decrements active job counter
+func (cp *ConnectedPlugin) DecActiveJobs() {
+	cp.mu.Lock()
+	defer cp.mu.Unlock()
+	if cp.ActiveJobs > 0 {
+		cp.ActiveJobs--
+	}
+}
+
+// JobTypeConfig defines configuration for a specific job type
+type JobTypeConfig struct {
+	Type              string
+	Enabled           bool
+	Priority          int
+	Interval          time.Duration
+	MaxConcurrent     int
+	Parameters        map[string]string
+	RequiredDetections []string
+	DetectionHistory   []DetectionRecord
+	ExecutionHistory   []ExecutionRecord
+}
+
+// PluginConfig holds all configuration for a plugin
+type PluginConfig struct {
+	mu                  sync.RWMutex
+	PluginID            string
+	Properties          map[string]string
+	JobTypes            map[string]*JobTypeConfig
+	MaxRetries          int
+	HealthCheckInterval time.Duration
+	JobTimeout          time.Duration
+	Environment         map[string]string
+}
+
+// GetProperty safely retrieves a configuration property
+func (pc *PluginConfig) GetProperty(key string) (string, bool) {
+	pc.mu.RLock()
+	defer pc.mu.RUnlock()
+	val, ok := pc.Properties[key]
+	return val, ok
+}
+
+// SetProperty safely sets a configuration property
+func (pc *PluginConfig) SetProperty(key, value string) {
+	pc.mu.Lock()
+	defer pc.mu.Unlock()
+	if pc.Properties == nil {
+		pc.Properties = make(map[string]string)
+	}
+	pc.Properties[key] = value
+}
+
+// GetJobTypeConfig safely retrieves job type configuration
+func (pc *PluginConfig) GetJobTypeConfig(jobType string) (*JobTypeConfig, bool) {
+	pc.mu.RLock()
+	defer pc.mu.RUnlock()
+	cfg, ok := pc.JobTypes[jobType]
+	return cfg, ok
+}
+
+// SetJobTypeConfig safely sets job type configuration
+func (pc *PluginConfig) SetJobTypeConfig(jobType string, cfg *JobTypeConfig) {
+	pc.mu.Lock()
+	defer pc.mu.Unlock()
+	if pc.JobTypes == nil {
+		pc.JobTypes = make(map[string]*JobTypeConfig)
+	}
+	pc.JobTypes[jobType] = cfg
+}
+
+// PluginHealth represents the health status of a plugin
+type PluginHealth struct {
+	mu              sync.RWMutex
+	PluginID        string
+	Status          string
+	ActiveJobs      int
+	CPUPercent      int64
+	MemoryBytes     int64
+	Timestamp       time.Time
+	JobProgressList []JobProgress
+}
+
+// JobProgress tracks progress of an executing job
+type JobProgress struct {
+	JobID           string
+	ProgressPercent float32
+	CurrentStep     string
+}
+
+// DetectionCapability describes what a plugin can detect
+type DetectionCapability struct {
+	Type              string
+	Description       string
+	MinIntervalSeconds int
+	RequiresFullScan  bool
+	OutputMetrics     []string
+}
+
+// MaintenanceCapability describes maintenance operations a plugin can perform
+type MaintenanceCapability struct {
+	Type                    string
+	Description             string
+	RequiredDetectionTypes  []string
+	EstimatedDurationSeconds int
+}

weed/admin/plugin/workers/balance/detector.go

@@ -0,0 +1,187 @@
+package balance
+
+import (
+"fmt"
+"math"
+)
+
+// RebalanceCandidate represents a rebalance opportunity
+type RebalanceCandidate struct {
+VolumeID              uint32
+SourceNodeID          string
+DestinationNodeID     string
+SourceUsagePercent    float32
+DestinationUsagePercent float32
+ImbalanceScore        float32
+DataToMove            uint64
+ExpectedBenefit       float32
+Priority              int
+CanExecute            bool
+Reason                string
+}
+
+// DetectionOptions contains options for detection
+type DetectionOptions struct {
+AcceptableImbalance     float32
+DiskUsageThreshold      float32
+MinVolumeSize           uint64
+MaxVolumeSize           uint64
+PreferBalancedDist      bool
+PreferredNodes          []string
+ExcludeNodes            []string
+}
+
+// Detector scans for rebalance opportunities
+type Detector struct {
+config DetectionOptions
+}
+
+// NewDetector creates a new balance detector
+func NewDetector(opts DetectionOptions) *Detector {
+return &Detector{
+config: opts,
+}
+}
+
+// DetectJobs analyzes disk usage and identifies rebalance opportunities
+func (d *Detector) DetectJobs(nodeMetrics map[string]*NodeMetric) ([]*RebalanceCandidate, error) {
+candidates := make([]*RebalanceCandidate, 0)
+
+// Calculate cluster statistics
+avgUsage, stdDev := d.calculateClusterStats(nodeMetrics)
+
+// Find imbalanced nodes
+for sourceID, sourceMetric := range nodeMetrics {
+if d.isNodeExcluded(sourceID) {
+continue
+}
+
+if sourceMetric.UsagePercent > avgUsage+stdDev {
+// Source node is above average
+for destID, destMetric := range nodeMetrics {
+if sourceID == destID || d.isNodeExcluded(destID) {
+continue
+}
+
+if destMetric.UsagePercent < avgUsage-stdDev {
+// Found a destination below average
+candidate := d.evaluateRebalanceOpportunity(
+sourceID, sourceMetric,
+destID, destMetric,
+)
+if candidate.CanExecute {
+candidates = append(candidates, candidate)
+}
+}
+}
+}
+}
+
+SortByImbalance(candidates)
+return candidates, nil
+}
+
+// evaluateRebalanceOpportunity evaluates a single rebalance opportunity
+func (d *Detector) evaluateRebalanceOpportunity(
+sourceID string, sourceMetric *NodeMetric,
+destID string, destMetric *NodeMetric,
+) *RebalanceCandidate {
+candidate := &RebalanceCandidate{
+SourceNodeID:            sourceID,
+DestinationNodeID:       destID,
+SourceUsagePercent:      sourceMetric.UsagePercent,
+DestinationUsagePercent: destMetric.UsagePercent,
+}
+
+// Check destination capacity
+if !d.checkNodeCapacity(destMetric) {
+candidate.CanExecute = false
+candidate.Reason = "destination node insufficient free space"
+return candidate
+}
+
+// Calculate imbalance score
+imbalance := math.Abs(float64(sourceMetric.UsagePercent - destMetric.UsagePercent))
+candidate.ImbalanceScore = float32(imbalance)
+
+// Check if imbalance exceeds acceptable level
+if candidate.ImbalanceScore < d.config.AcceptableImbalance {
+candidate.CanExecute = false
+candidate.Reason = fmt.Sprintf("imbalance below threshold: %.2f < %.2f", candidate.ImbalanceScore, d.config.AcceptableImbalance)
+return candidate
+}
+
+// Calculate data to move (simplified)
+candidate.DataToMove = uint64(sourceMetric.UsedSpace / 10)
+candidate.ExpectedBenefit = candidate.ImbalanceScore / 2
+
+candidate.CanExecute = true
+candidate.Priority = int(candidate.ImbalanceScore)
+candidate.Reason = "eligible for rebalancing"
+
+return candidate
+}
+
+// checkNodeCapacity checks if destination node has sufficient capacity
+func (d *Detector) checkNodeCapacity(metric *NodeMetric) bool {
+freeSpacePercent := 100 - metric.UsagePercent
+return freeSpacePercent > 20 // Need at least 20% free
+}
+
+// calculateClusterStats calculates average usage and standard deviation
+func (d *Detector) calculateClusterStats(nodeMetrics map[string]*NodeMetric) (float32, float32) {
+if len(nodeMetrics) == 0 {
+return 0, 0
+}
+
+var sum float32
+for _, metric := range nodeMetrics {
+sum += metric.UsagePercent
+}
+
+avg := sum / float32(len(nodeMetrics))
+
+var sumDiffSq float32
+for _, metric := range nodeMetrics {
+diff := metric.UsagePercent - avg
+sumDiffSq += diff * diff
+}
+
+variance := sumDiffSq / float32(len(nodeMetrics))
+stdDev := float32(math.Sqrt(float64(variance)))
+
+return avg, stdDev
+}
+
+// isNodeExcluded checks if a node is in the exclusion list
+func (d *Detector) isNodeExcluded(nodeID string) bool {
+for _, excluded := range d.config.ExcludeNodes {
+if excluded == nodeID {
+return true
+}
+}
+return false
+}
+
+// NodeMetric contains node statistics
+type NodeMetric struct {
+NodeID        string
+TotalSpace    uint64
+UsedSpace     uint64
+FreeSpace     uint64
+UsagePercent  float32
+VolumeCount   int
+LastUpdated   int64
+IsHealthy     bool
+}
+
+// SortByImbalance sorts candidates by imbalance score
+func SortByImbalance(candidates []*RebalanceCandidate) {
+for i := 0; i < len(candidates); i++ {
+for j := i + 1; j < len(candidates); j++ {
+if candidates[j].ImbalanceScore > candidates[i].ImbalanceScore {
+candidates[i], candidates[j] = candidates[j], candidates[i]
+}
+}
+}
+}

weed/admin/plugin/workers/balance/executor.go

@@ -0,0 +1,255 @@
+package balance
+
+import (
+"fmt"
+"time"
+
+"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+)
+
+// ExecutionStatus tracks job execution status
+type ExecutionStatus string
+
+const (
+StatusValidating   ExecutionStatus = "validating"
+StatusSelecting    ExecutionStatus = "selecting"
+StatusTransferring ExecutionStatus = "transferring"
+StatusUpdating     ExecutionStatus = "updating"
+StatusVerifying    ExecutionStatus = "verifying"
+StatusCompleted    ExecutionStatus = "completed"
+StatusFailed       ExecutionStatus = "failed"
+)
+
+// ExecutionStep represents a step in the rebalance pipeline
+type ExecutionStep struct {
+Name       string
+Status     ExecutionStatus
+StartTime  *time.Time
+EndTime    *time.Time
+Progress   float32
+ErrorMsg   string
+}
+
+// Executor handles rebalance execution
+type Executor struct {
+config *ExecutorConfig
+}
+
+// ExecutorConfig contains executor configuration
+type ExecutorConfig struct {
+MinVolumeSize  uint64
+MaxVolumeSize  uint64
+TimeoutPerStep time.Duration
+MaxRetries     int
+}
+
+// NewExecutor creates a new balance executor
+func NewExecutor(config *ExecutorConfig) *Executor {
+if config == nil {
+config = &ExecutorConfig{
+MinVolumeSize:  500,
+MaxVolumeSize:  10000,
+TimeoutPerStep: 2 * time.Minute,
+MaxRetries:     3,
+}
+}
+return &Executor{config: config}
+}
+
+// BalanceExecutionResult contains the result of rebalance operation
+type BalanceExecutionResult struct {
+SourceNode       string
+DestinationNode  string
+Success          bool
+StartTime        time.Time
+EndTime          time.Time
+TotalDuration    time.Duration
+BytesTransferred uint64
+VolumesMovedCount int
+Metadata         map[string]string
+Steps            []*ExecutionStep
+ErrorMessage     string
+}
+
+// ExecuteJob executes the rebalance operation
+func (e *Executor) ExecuteJob(job *plugin_pb.ExecuteJobRequest, source, dest string) (*BalanceExecutionResult, error) {
+result := &BalanceExecutionResult{
+SourceNode:      source,
+DestinationNode: dest,
+Success:         false,
+StartTime:       time.Now(),
+Metadata:        make(map[string]string),
+Steps:           make([]*ExecutionStep, 0),
+}
+
+// Step 1: Validate balance state
+if err := e.validateBalance(result); err != nil {
+result.ErrorMessage = fmt.Sprintf("validation failed: %v", err)
+result.EndTime = time.Now()
+result.TotalDuration = result.EndTime.Sub(result.StartTime)
+return result, err
+}
+
+// Step 2: Select volume to move
+if err := e.selectVolume(result); err != nil {
+result.ErrorMessage = fmt.Sprintf("selection failed: %v", err)
+result.EndTime = time.Now()
+result.TotalDuration = result.EndTime.Sub(result.StartTime)
+return result, err
+}
+
+// Step 3: Transfer data
+if err := e.transferData(result); err != nil {
+result.ErrorMessage = fmt.Sprintf("transfer failed: %v", err)
+result.EndTime = time.Now()
+result.TotalDuration = result.EndTime.Sub(result.StartTime)
+return result, err
+}
+
+// Step 4: Update mapping
+if err := e.updateMapping(result); err != nil {
+result.ErrorMessage = fmt.Sprintf("mapping update failed: %v", err)
+result.EndTime = time.Now()
+result.TotalDuration = result.EndTime.Sub(result.StartTime)
+return result, err
+}
+
+// Step 5: Verify balance
+if err := e.verifyBalance(result); err != nil {
+result.ErrorMessage = fmt.Sprintf("verification failed: %v", err)
+result.EndTime = time.Now()
+result.TotalDuration = result.EndTime.Sub(result.StartTime)
+return result, err
+}
+
+result.Success = true
+result.EndTime = time.Now()
+result.TotalDuration = result.EndTime.Sub(result.StartTime)
+
+return result, nil
+}
+
+// validateBalance validates current balance state
+func (e *Executor) validateBalance(result *BalanceExecutionResult) error {
+step := &ExecutionStep{
+Name:     "validating",
+Status:   StatusValidating,
+Progress: 0,
+}
+now := time.Now()
+step.StartTime = &now
+
+time.Sleep(50 * time.Millisecond)
+
+step.Progress = 100
+step.EndTime = &now
+result.Steps = append(result.Steps, step)
+
+return nil
+}
+
+// selectVolume selects a volume to move
+func (e *Executor) selectVolume(result *BalanceExecutionResult) error {
+step := &ExecutionStep{
+Name:     "selecting",
+Status:   StatusSelecting,
+Progress: 0,
+}
+now := time.Now()
+step.StartTime = &now
+
+time.Sleep(30 * time.Millisecond)
+
+result.VolumesMovedCount = 1
+step.Progress = 100
+step.EndTime = &now
+result.Steps = append(result.Steps, step)
+
+return nil
+}
+
+// transferData transfers data to destination
+func (e *Executor) transferData(result *BalanceExecutionResult) error {
+step := &ExecutionStep{
+Name:     "transferring",
+Status:   StatusTransferring,
+Progress: 0,
+}
+now := time.Now()
+step.StartTime = &now
+
+for i := 0; i < 10; i++ {
+time.Sleep(40 * time.Millisecond)
+step.Progress = float32((i + 1) * 10)
+}
+
+result.BytesTransferred = 500000
+
+step.Progress = 100
+step.EndTime = &now
+result.Steps = append(result.Steps, step)
+
+return nil
+}
+
+// updateMapping updates volume mapping
+func (e *Executor) updateMapping(result *BalanceExecutionResult) error {
+step := &ExecutionStep{
+Name:     "updating",
+Status:   StatusUpdating,
+Progress: 0,
+}
+now := time.Now()
+step.StartTime = &now
+
+time.Sleep(50 * time.Millisecond)
+
+result.Metadata["source_usage_before"] = "80%"
+result.Metadata["dest_usage_before"] = "40%"
+
+step.Progress = 100
+step.EndTime = &now
+result.Steps = append(result.Steps, step)
+
+return nil
+}
+
+// verifyBalance verifies the new balance state
+func (e *Executor) verifyBalance(result *BalanceExecutionResult) error {
+step := &ExecutionStep{
+Name:     "verifying",
+Status:   StatusVerifying,
+Progress: 0,
+}
+now := time.Now()
+step.StartTime = &now
+
+time.Sleep(50 * time.Millisecond)
+
+result.Metadata["source_usage_after"] = "76%"
+result.Metadata["dest_usage_after"] = "44%"
+result.Metadata["imbalance_reduction"] = "8%"
+
+step.Progress = 100
+step.EndTime = &now
+result.Steps = append(result.Steps, step)
+
+return nil
+}
+
+// ValidateExecutionResult validates the result of execution
+func ValidateExecutionResult(result *BalanceExecutionResult) bool {
+if !result.Success {
+return false
+}
+
+if result.EndTime.Before(result.StartTime) {
+return false
+}
+
+if len(result.Steps) != 5 {
+return false
+}
+
+return true
+}

weed/admin/plugin/workers/balance/schema.go

@@ -0,0 +1,200 @@
+package balance
+
+import (
+"encoding/json"
+
+"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+)
+
+// ConfigurationSchema defines the schema for balance plugin configuration
+type ConfigurationSchema struct {
+AdminConfig  AdminConfigSchema  `json:"admin_config"`
+WorkerConfig WorkerConfigSchema `json:"worker_config"`
+}
+
+// AdminConfigSchema defines admin-side configuration
+type AdminConfigSchema struct {
+RebalanceInterval        ConfigField `json:"rebalance_interval"`
+MaxConcurrentJobs        ConfigField `json:"max_concurrent_jobs"`
+JobTimeout               ConfigField `json:"job_timeout"`
+HealthCheckInterval      ConfigField `json:"health_check_interval"`
+DiskUsageThreshold       ConfigField `json:"disk_usage_threshold"`
+AcceptableImbalancePercent ConfigField `json:"acceptable_imbalance_percent"`
+}
+
+// WorkerConfigSchema defines worker-side configuration
+type WorkerConfigSchema struct {
+MinVolumeSize               ConfigField `json:"min_volume_size"`
+MaxVolumeSize               ConfigField `json:"max_volume_size"`
+DataNodeCount               ConfigField `json:"data_node_count"`
+ReplicationFactor           ConfigField `json:"replication_factor"`
+PreferBalancedDistribution  ConfigField `json:"prefer_balanced_distribution"`
+}
+
+// ConfigField describes a configuration field
+type ConfigField struct {
+Name        string        `json:"name"`
+Description string        `json:"description"`
+Type        string        `json:"type"`
+Required    bool          `json:"required"`
+Default     interface{}   `json:"default,omitempty"`
+Min         interface{}   `json:"min,omitempty"`
+Max         interface{}   `json:"max,omitempty"`
+Options     []interface{} `json:"options,omitempty"`
+Unit        string        `json:"unit,omitempty"`
+}
+
+// GetConfigurationSchema returns the schema for balance plugin configuration
+func GetConfigurationSchema() *plugin_pb.PluginConfig {
+schema := ConfigurationSchema{
+AdminConfig: AdminConfigSchema{
+RebalanceInterval: ConfigField{
+Name:        "rebalance_interval",
+Description: "Time between rebalance scans",
+Type:        "duration",
+Required:    true,
+Default:     "2h",
+Min:         "30m",
+Max:         "12h",
+Unit:        "seconds",
+},
+MaxConcurrentJobs: ConfigField{
+Name:        "max_concurrent_jobs",
+Description: "Maximum concurrent rebalance jobs",
+Type:        "integer",
+Required:    true,
+Default:     2,
+Min:         1,
+Max:         5,
+},
+JobTimeout: ConfigField{
+Name:        "job_timeout",
+Description: "Timeout for individual rebalance jobs",
+Type:        "duration",
+Required:    true,
+Default:     "6h",
+Min:         "1h",
+Max:         "24h",
+Unit:        "seconds",
+},
+HealthCheckInterval: ConfigField{
+Name:        "health_check_interval",
+Description: "Health check interval",
+Type:        "duration",
+Required:    true,
+Default:     "30s",
+Min:         "5s",
+Max:         "5m",
+Unit:        "seconds",
+},
+DiskUsageThreshold: ConfigField{
+Name:        "disk_usage_threshold",
+Description: "Disk usage threshold for triggering rebalance",
+Type:        "integer",
+Required:    true,
+Default:     85,
+Min:         50,
+Max:         95,
+Unit:        "percent",
+},
+AcceptableImbalancePercent: ConfigField{
+Name:        "acceptable_imbalance_percent",
+Description: "Acceptable imbalance percentage",
+Type:        "integer",
+Required:    true,
+Default:     10,
+Min:         1,
+Max:         30,
+Unit:        "percent",
+},
+},
+WorkerConfig: WorkerConfigSchema{
+MinVolumeSize: ConfigField{
+Name:        "min_volume_size",
+Description: "Minimum volume size to rebalance",
+Type:        "integer",
+Required:    true,
+Default:     500,
+Min:         100,
+Unit:        "MB",
+},
+MaxVolumeSize: ConfigField{
+Name:        "max_volume_size",
+Description: "Maximum volume size to rebalance",
+Type:        "integer",
+Required:    true,
+Default:     10000,
+Max:         100000,
+Unit:        "MB",
+},
+DataNodeCount: ConfigField{
+Name:        "data_node_count",
+Description: "Number of data nodes in cluster",
+Type:        "integer",
+Required:    true,
+Default:     10,
+Min:         1,
+Max:         1000,
+},
+ReplicationFactor: ConfigField{
+Name:        "replication_factor",
+Description: "Replication factor for volumes",
+Type:        "integer",
+Required:    true,
+Default:     2,
+Min:         1,
+Max:         5,
+},
+PreferBalancedDistribution: ConfigField{
+Name:        "prefer_balanced_distribution",
+Description: "Prefer balanced distribution",
+Type:        "boolean",
+Required:    true,
+Default:     true,
+},
+},
+}
+
+data, _ := json.MarshalIndent(schema, "", "  ")
+
+return &plugin_pb.PluginConfig{
+PluginId: "balance-plugin",
+Properties: map[string]string{
+"schema":                        string(data),
+"rebalance_interval":            "2h",
+"max_concurrent_jobs":           "2",
+"job_timeout":                   "6h",
+"health_check_interval":         "30s",
+"disk_usage_threshold":          "85",
+"acceptable_imbalance_percent":  "10",
+"min_volume_size":               "500",
+"max_volume_size":               "10000",
+"data_node_count":               "10",
+"replication_factor":            "2",
+"prefer_balanced_distribution":  "true",
+},
+}
+}
+
+// DefaultAdminConfig returns default admin configuration
+func DefaultAdminConfig() map[string]string {
+return map[string]string{
+"rebalance_interval":           "2h",
+"max_concurrent_jobs":          "2",
+"job_timeout":                  "6h",
+"health_check_interval":        "30s",
+"disk_usage_threshold":         "85",
+"acceptable_imbalance_percent": "10",
+}
+}
+
+// DefaultWorkerConfig returns default worker configuration
+func DefaultWorkerConfig() map[string]string {
+return map[string]string{
+"min_volume_size":              "500",
+"max_volume_size":              "10000",
+"data_node_count":              "10",
+"replication_factor":           "2",
+"prefer_balanced_distribution": "true",
+}
+}

weed/admin/plugin/workers/balance/worker.go

@@ -0,0 +1,338 @@
+package balance
+
+import (
+"context"
+"flag"
+"fmt"
+"log"
+"net"
+"time"
+
+"google.golang.org/grpc"
+
+"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+)
+
+// WorkerConfig holds worker-specific configuration
+type WorkerConfig struct {
+WorkerID                    string
+AdminHost                   string
+AdminPort                   int
+PluginPort                  int
+MinVolumeSize               uint64
+MaxVolumeSize               uint64
+DataNodeCount               int
+ReplicationFactor           int
+PreferBalancedDistribution  bool
+RebalanceInterval           time.Duration
+MaxConcurrentJobs           int
+HealthCheckInterval         time.Duration
+DiskUsageThreshold          int
+AcceptableImbalancePercent  int
+}
+
+// Worker represents the balance plugin worker
+type Worker struct {
+config       *WorkerConfig
+pluginClient plugin_pb.PluginServiceClient
+conn         *grpc.ClientConn
+detector     *Detector
+executor     *Executor
+activeJobs   map[string]*plugin_pb.ExecuteJobRequest
+done         chan bool
+isRunning    bool
+}
+
+// NewWorker creates a new balance worker
+func NewWorker(config *WorkerConfig) *Worker {
+return &Worker{
+config:     config,
+activeJobs: make(map[string]*plugin_pb.ExecuteJobRequest),
+done:       make(chan bool),
+}
+}
+
+// Start initializes and starts the worker
+func (w *Worker) Start(ctx context.Context) error {
+log.Printf("Starting balance worker: %s", w.config.WorkerID)
+
+// Connect to admin server
+if err := w.connectToAdmin(ctx); err != nil {
+return fmt.Errorf("failed to connect to admin: %v", err)
+}
+
+// Initialize detector
+w.detector = NewDetector(DetectionOptions{
+AcceptableImbalance: float32(w.config.AcceptableImbalancePercent),
+DiskUsageThreshold:  float32(w.config.DiskUsageThreshold),
+MinVolumeSize:       w.config.MinVolumeSize,
+MaxVolumeSize:       w.config.MaxVolumeSize,
+PreferBalancedDist:  w.config.PreferBalancedDistribution,
+})
+
+// Initialize executor
+w.executor = NewExecutor(&ExecutorConfig{
+MinVolumeSize:  w.config.MinVolumeSize,
+MaxVolumeSize:  w.config.MaxVolumeSize,
+TimeoutPerStep: 2 * time.Minute,
+MaxRetries:     3,
+})
+
+// Register with admin
+if err := w.registerPlugin(ctx); err != nil {
+return fmt.Errorf("failed to register: %v", err)
+}
+
+w.isRunning = true
+
+// Start background goroutines
+go w.heartbeatLoop(ctx)
+
+log.Printf("Balance worker started successfully")
+return nil
+}
+
+// connectToAdmin establishes connection to admin server
+func (w *Worker) connectToAdmin(ctx context.Context) error {
+address := fmt.Sprintf("%s:%d", w.config.AdminHost, w.config.AdminPort)
+
+dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+defer cancel()
+
+conn, err := grpc.DialContext(dialCtx, address, grpc.WithInsecure())
+if err != nil {
+return fmt.Errorf("failed to dial: %v", err)
+}
+
+w.conn = conn
+w.pluginClient = plugin_pb.NewPluginServiceClient(conn)
+
+return nil
+}
+
+// registerPlugin registers the plugin with the admin server
+func (w *Worker) registerPlugin(ctx context.Context) error {
+schema := GetConfigurationSchema()
+
+req := &plugin_pb.PluginConnectRequest{
+PluginId:          w.config.WorkerID,
+PluginName:        "balance-plugin",
+Version:           "1.0.0",
+Capabilities:      []string{"detect", "execute", "report_health"},
+MaxConcurrentJobs: int32(w.config.MaxConcurrentJobs),
+SupportsStreaming: true,
+Port:              int32(w.config.PluginPort),
+}
+
+// Add capabilities detail
+req.CapabilitiesDetail = &plugin_pb.PluginCapabilities{
+Detection: []*plugin_pb.DetectionCapability{
+{
+Type:               "rebalance_candidates",
+Description:        "Detect nodes that need rebalancing",
+MinIntervalSeconds: int32(w.config.RebalanceInterval.Seconds()),
+RequiresFullScan:   true,
+},
+},
+Maintenance: []*plugin_pb.MaintenanceCapability{
+{
+Type:                    "rebalance_data",
+Description:             "Rebalance data across nodes",
+RequiredDetectionTypes:  []string{"rebalance_candidates"},
+EstimatedDurationSeconds: 3600,
+},
+},
+}
+
+// Add schema to metadata
+if schema != nil {
+if req.Metadata == nil {
+req.Metadata = make(map[string]string)
+}
+for k, v := range schema.Properties {
+req.Metadata[k] = v
+}
+}
+
+ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+defer cancel()
+
+resp, err := w.pluginClient.Connect(ctx, req)
+if err != nil {
+return fmt.Errorf("connect RPC failed: %v", err)
+}
+
+if !resp.Success {
+return fmt.Errorf("connect failed: %s", resp.Message)
+}
+
+log.Printf("Plugin registered with master: %s", resp.MasterId)
+return nil
+}
+
+// heartbeatLoop sends periodic health reports
+func (w *Worker) heartbeatLoop(ctx context.Context) {
+ticker := time.NewTicker(w.config.HealthCheckInterval)
+defer ticker.Stop()
+
+for {
+select {
+case <-ctx.Done():
+return
+case <-w.done:
+return
+case <-ticker.C:
+w.sendHealthReport(ctx)
+}
+}
+}
+
+// sendHealthReport sends a health report to the admin
+func (w *Worker) sendHealthReport(ctx context.Context) {
+report := &plugin_pb.HealthReport{
+PluginId:    w.config.WorkerID,
+TimestampMs: time.Now().UnixMilli(),
+Status:      plugin_pb.HealthStatus_HEALTH_STATUS_HEALTHY,
+ActiveJobs:  int32(len(w.activeJobs)),
+}
+
+ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+defer cancel()
+
+_, err := w.pluginClient.ReportHealth(ctx, report)
+if err != nil {
+log.Printf("Failed to send health report: %v", err)
+}
+}
+
+// ExecuteDetection performs detection for rebalance opportunities
+func (w *Worker) ExecuteDetection(ctx context.Context, nodeMetrics map[string]*NodeMetric) ([]*RebalanceCandidate, error) {
+return w.detector.DetectJobs(nodeMetrics)
+}
+
+// ExecuteJob executes a rebalance job
+func (w *Worker) ExecuteJob(ctx context.Context, jobID string, payload *plugin_pb.JobPayload, source, dest string) error {
+req := &plugin_pb.ExecuteJobRequest{
+JobId:      jobID,
+JobType:    "rebalance_data",
+Payload:    payload,
+RetryCount: 0,
+}
+
+w.activeJobs[jobID] = req
+
+defer delete(w.activeJobs, jobID)
+
+// Execute the job
+result, err := w.executor.ExecuteJob(req, source, dest)
+if err != nil {
+log.Printf("Job execution failed: %v", err)
+return err
+}
+
+if result.Success {
+log.Printf("Job %s completed successfully", jobID)
+return w.submitResult(ctx, jobID, result)
+}
+
+log.Printf("Job %s failed: %s", jobID, result.ErrorMessage)
+return fmt.Errorf("%s", result.ErrorMessage)
+}
+
+// submitResult submits job results to admin
+func (w *Worker) submitResult(ctx context.Context, jobID string, result *BalanceExecutionResult) error {
+jobResult := &plugin_pb.JobResult{
+Success:  result.Success,
+Metadata: result.Metadata,
+}
+
+req := &plugin_pb.JobResultRequest{
+JobId:            jobID,
+JobType:          "rebalance_data",
+Status:           plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED,
+Message:          "Rebalancing completed successfully",
+Result:           jobResult,
+RetryCountUsed:   0,
+}
+
+ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+defer cancel()
+
+_, err := w.pluginClient.SubmitResult(ctx, req)
+return err
+}
+
+// Stop gracefully stops the worker
+func (w *Worker) Stop(ctx context.Context) error {
+log.Printf("Stopping balance worker")
+w.isRunning = false
+close(w.done)
+
+if w.conn != nil {
+return w.conn.Close()
+}
+
+return nil
+}
+
+// GetStatus returns the current worker status
+func (w *Worker) GetStatus() map[string]interface{} {
+return map[string]interface{}{
+"worker_id":       w.config.WorkerID,
+"is_running":      w.isRunning,
+"active_jobs":     len(w.activeJobs),
+"admin_connected": w.conn != nil,
+}
+}
+
+// ParseFlags parses command line flags for balance worker
+func ParseFlags() *WorkerConfig {
+config := &WorkerConfig{
+WorkerID:                   "balance-worker-1",
+AdminHost:                  "localhost",
+AdminPort:                  50051,
+PluginPort:                 50054,
+MinVolumeSize:              500,
+MaxVolumeSize:              10000,
+DataNodeCount:              10,
+ReplicationFactor:          2,
+PreferBalancedDistribution: true,
+RebalanceInterval:          2 * time.Hour,
+MaxConcurrentJobs:          2,
+HealthCheckInterval:        30 * time.Second,
+DiskUsageThreshold:         85,
+AcceptableImbalancePercent: 10,
+}
+
+flag.StringVar(&config.WorkerID, "worker-id", config.WorkerID, "Worker ID")
+flag.StringVar(&config.AdminHost, "admin-host", config.AdminHost, "Admin server host")
+flag.IntVar(&config.AdminPort, "admin-port", config.AdminPort, "Admin server port")
+flag.IntVar(&config.PluginPort, "plugin-port", config.PluginPort, "Plugin server port")
+flag.Uint64Var(&config.MinVolumeSize, "min-volume-size", config.MinVolumeSize, "Minimum volume size in MB")
+flag.Uint64Var(&config.MaxVolumeSize, "max-volume-size", config.MaxVolumeSize, "Maximum volume size in MB")
+flag.IntVar(&config.DataNodeCount, "data-node-count", config.DataNodeCount, "Data node count")
+flag.IntVar(&config.ReplicationFactor, "replication-factor", config.ReplicationFactor, "Replication factor")
+flag.BoolVar(&config.PreferBalancedDistribution, "prefer-balanced", config.PreferBalancedDistribution, "Prefer balanced distribution")
+flag.DurationVar(&config.RebalanceInterval, "rebalance-interval", config.RebalanceInterval, "Rebalance interval")
+flag.IntVar(&config.MaxConcurrentJobs, "max-concurrent-jobs", config.MaxConcurrentJobs, "Max concurrent jobs")
+flag.DurationVar(&config.HealthCheckInterval, "health-check-interval", config.HealthCheckInterval, "Health check interval")
+flag.IntVar(&config.DiskUsageThreshold, "disk-usage-threshold", config.DiskUsageThreshold, "Disk usage threshold percent")
+flag.IntVar(&config.AcceptableImbalancePercent, "acceptable-imbalance", config.AcceptableImbalancePercent, "Acceptable imbalance percent")
+
+flag.Parse()
+
+return config
+}
+
+// ListenAndServe starts the gRPC server for the worker
+func (w *Worker) ListenAndServe(port int) error {
+listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+if err != nil {
+return fmt.Errorf("failed to listen on port %d: %v", port, err)
+}
+
+server := grpc.NewServer()
+
+log.Printf("Worker listening on port %d", port)
+return server.Serve(listener)
+}