docs(volume_server): use repo-relative paths in plan/readme

.github/workflows/java_integration_tests.yml

@@ -111,7 +111,7 @@ jobs:
           
           # Wait for S3 API
           for i in {1..30}; do
-            if curl -s http://localhost:8333/healthz > /dev/null 2>&1; then
+            if curl -s http://localhost:8333/ > /dev/null 2>&1; then
               echo "✓ S3 API is ready"
               break
             fi

.github/workflows/s3-example-integration-tests.yml

@@ -39,14 +39,6 @@ jobs:
           echo "=== Running S3 Integration Tests ==="
           go test -v -timeout=60s -run TestS3Integration ./...
 
-      - name: Run S3 DeleteBucketNotEmpty Tests
-        timeout-minutes: 15
-        working-directory: test/s3/normal
-        run: |
-          set -x
-          echo "=== Running S3 DeleteBucketNotEmpty Tests ==="
-          go test -v -timeout=60s -run TestS3DeleteBucketNotEmpty ./...
-
       - name: Run IAM Integration Tests
         timeout-minutes: 15
         working-directory: test/s3/normal

.github/workflows/s3-go-tests.yml

@@ -461,8 +461,6 @@ jobs:
           export S3_ENDPOINT="http://localhost:8006"
           export S3_ACCESS_KEY="0555b35654ad1656d804"
           export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
-          export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY"
-          export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY"
           
           # Run the specific test that is equivalent to AWS S3 tagging behavior
           make test-with-server || {

.github/workflows/s3-keycloak-tests.yml

@@ -97,7 +97,7 @@ jobs:
           # Verify service accessibility
           echo "=== Verifying Service Accessibility ==="
           curl -f http://localhost:8080/realms/master
-          curl -s http://localhost:8333/healthz
+          curl -s http://localhost:8333
           echo "✅ SeaweedFS S3 API is responding (IAM-protected endpoint)"
           
           # Run Keycloak-specific tests

.github/workflows/test-s3-over-https-using-awscli.yml

@@ -35,7 +35,7 @@ jobs:
           set -e
           mkdir -p /tmp/data
           ./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
-          until curl -s http://localhost:8333/healthz > /dev/null; do sleep 1; done
+          until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
 
       - name: Setup Caddy
         run: |
@@ -54,7 +54,7 @@ jobs:
       - name: Start Caddy
         run: |
           ./caddy start
-          until curl -fsS --insecure https://localhost:8443/healthz > /dev/null; do sleep 1; done
+          until curl -fsS --insecure https://localhost:8443 > /dev/null; do sleep 1; done
 
       - name: Create Bucket
         run: |

.github/workflows/volume-server-integration-tests.yml

@@ -9,6 +9,7 @@ on:
       - 'weed/storage/**'
       - 'weed/pb/volume_server.proto'
       - 'weed/pb/volume_server_pb/**'
+      - 'rust/volume_server/**'
       - '.github/workflows/volume-server-integration-tests.yml'
   push:
     branches: [ master, main ]
@@ -18,6 +19,7 @@ on:
       - 'weed/storage/**'
       - 'weed/pb/volume_server.proto'
       - 'weed/pb/volume_server_pb/**'
+      - 'rust/volume_server/**'
       - '.github/workflows/volume-server-integration-tests.yml'
 
 concurrency:
@@ -120,3 +122,55 @@ jobs:
           echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
           echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
           echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
+
+  volume-server-rust-smoke:
+    name: Volume Server Rust Smoke (${{ matrix.mode }})
+    runs-on: ubuntu-22.04
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: [exec, proxy, native]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v6
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Set up Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Build SeaweedFS binary
+        run: |
+          cd weed
+          go build -o weed .
+          chmod +x weed
+          ./weed version
+
+      - name: Run Rust-mode HTTP smoke tests
+        env:
+          WEED_BINARY: ${{ github.workspace }}/weed/weed
+          VOLUME_SERVER_IMPL: rust
+          VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
+        run: |
+          go test -v -count=1 -timeout=25m ./test/volume_server/http/... -run "TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip"
+
+      - name: Run Rust-mode gRPC smoke tests
+        env:
+          WEED_BINARY: ${{ github.workspace }}/weed/weed
+          VOLUME_SERVER_IMPL: rust
+          VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
+        run: |
+          go test -v -count=1 -timeout=25m ./test/volume_server/grpc/... -run "TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus"
+
+      - name: Rust smoke summary
+        if: always()
+        run: |
+          echo "## Volume Server Rust Smoke Summary (${{ matrix.mode }})" >> "$GITHUB_STEP_SUMMARY"
+          echo "- Mode: Go master + Rust volume server launcher (VOLUME_SERVER_IMPL=rust, VOLUME_SERVER_RUST_MODE=${{ matrix.mode }})" >> "$GITHUB_STEP_SUMMARY"
+          echo "- HTTP Command: go test -v -count=1 -timeout=25m ./test/volume_server/http/... -run \"TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip\"" >> "$GITHUB_STEP_SUMMARY"
+          echo "- gRPC Command: go test -v -count=1 -timeout=25m ./test/volume_server/grpc/... -run \"TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus\"" >> "$GITHUB_STEP_SUMMARY"

Makefile

@@ -40,7 +40,7 @@ test: admin-generate
 # Admin component targets
 admin-generate:
 	@echo "Generating admin component templates..."
-	@cd $(ADMIN_DIR) && templ generate ./view
+	@templ generate
 
 admin-build: admin-generate
 	@echo "Building admin component..."

docker/entrypoint.sh

@@ -20,17 +20,13 @@ if [ "$(id -u)" = "0" ]; then
   
   DATA_UID=$(stat -c '%u' /data 2>/dev/null)
   DATA_GID=$(stat -c '%g' /data 2>/dev/null)
-
-  # Only run chown -R if ownership doesn't already match (avoids expensive
-  # recursive chown on subsequent starts, and is a no-op on OpenShift when
-  # fsGroup has already set correct ownership on the PVC).
-  if [ "$DATA_UID" != "$SEAWEED_UID" ] || [ "$DATA_GID" != "$SEAWEED_GID" ]; then
+  
+  # Only run chown -R if ownership doesn't match (much faster for subsequent starts)
     echo "Fixing /data ownership for seaweed user (uid=$SEAWEED_UID, gid=$SEAWEED_GID)"
     if ! chown -R seaweed:seaweed /data; then
       echo "Warning: Failed to change ownership of /data. This may cause permission errors." >&2
       echo "If /data is read-only or has mount issues, the application may fail to start." >&2
     fi
-  fi
   
   # Use su-exec to drop privileges and run as seaweed user
   exec su-exec seaweed "$0" "$@"

docker/seaweedfs-dev-compose.yml

@@ -42,12 +42,3 @@ services:
       - master
       - volume
       - filer
-  admin:
-    image: chrislusf/seaweedfs:dev # use a remote dev image
-    ports:
-      - 23646:23646
-    command: 'admin -master=master:9333'
-    depends_on:
-      - master
-      - volume
-      - filer

go.mod

@@ -46,7 +46,7 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
-	github.com/klauspost/compress v1.18.4
+	github.com/klauspost/compress v1.18.3
 	github.com/klauspost/reedsolomon v1.13.0
 	github.com/kurin/blazer v0.5.3
 	github.com/linxGnu/grocksdb v1.10.7
@@ -130,7 +130,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
 	github.com/cognusion/imaging v1.0.2
 	github.com/fluent/fluent-logger-golang v1.10.1
-	github.com/getsentry/sentry-go v0.42.0
+	github.com/getsentry/sentry-go v0.40.0
 	github.com/gin-contrib/sessions v1.0.4
 	github.com/gin-gonic/gin v1.11.0
 	github.com/go-ldap/ldap/v3 v3.4.12
@@ -142,7 +142,7 @@ require (
 	github.com/jhump/protoreflect v1.18.0
 	github.com/lib/pq v1.11.1
 	github.com/linkedin/goavro/v2 v2.14.1
-	github.com/mattn/go-sqlite3 v1.14.34
+	github.com/mattn/go-sqlite3 v1.14.33
 	github.com/minio/crc64nvme v1.1.1
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/parquet-go/parquet-go v0.26.4
@@ -150,7 +150,7 @@ require (
 	github.com/rabbitmq/amqp091-go v1.10.0
 	github.com/rclone/rclone v1.72.1
 	github.com/rdleal/intervalst v1.5.0
-	github.com/redis/go-redis/v9 v9.18.0
+	github.com/redis/go-redis/v9 v9.17.2
 	github.com/schollz/progressbar/v3 v3.19.0
 	github.com/seaweedfs/go-fuse/v2 v2.9.1
 	github.com/shirou/gopsutil/v4 v4.26.1
@@ -293,11 +293,11 @@ require (
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	cloud.google.com/go/iam v1.5.3 // indirect
 	cloud.google.com/go/monitoring v1.24.2 // indirect
-	filippo.io/edwards25519 v1.1.1 // indirect
+	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
 	github.com/Azure/go-ntlmssp v0.1.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect

go.sum

@@ -547,8 +547,8 @@ cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcP
 dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
 dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw=
-filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8=
 git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
@@ -569,8 +569,8 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGlu
 github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 h1:sxgSqOB9CDToiaVFpxuvb5wGgGqWa3lCShcm5o0n3bE=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3/go.mod h1:XdED8i399lEVblYHTZM8eXaP07gv4Z58IL6ueMlVlrg=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
@@ -1053,8 +1053,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
 github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
 github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
 github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
-github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8=
-github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
+github.com/getsentry/sentry-go v0.40.0 h1:VTJMN9zbTvqDqPwheRVLcp0qcUcM+8eFivvGocAaSbo=
+github.com/getsentry/sentry-go v0.40.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sessions v1.0.4 h1:ha6CNdpYiTOK/hTp05miJLbpTSNfOnFg5Jm2kbcqy8U=
 github.com/gin-contrib/sessions v1.0.4/go.mod h1:ccmkrb2z6iU2osiAHZG3x3J4suJK+OU27oqzlWOqQgs=
@@ -1487,8 +1487,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
 github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
+github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
@@ -1564,8 +1564,8 @@ github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhg
 github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
 github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
-github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0=
+github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -1814,8 +1814,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
 github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
-github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
-github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
+github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
+github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
 github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
 github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
 github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=

k8s/charts/seaweedfs/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "4.13"
+appVersion: "4.12"
 # Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
-version: 4.0.413
+version: 4.0.412

k8s/charts/seaweedfs/README.md

@@ -212,9 +212,8 @@ To enable workers, add the following to your values.yaml:
 worker:
   enabled: true
   replicas: 2  # Scale based on workload
-  jobType: "vacuum,volume_balance,erasure_coding"  # Job types this worker can handle
-  maxDetect: 1  # Maximum concurrent detection requests
-  maxExecute: 4  # Maximum concurrent execution jobs per worker
+  capabilities: "vacuum,balance,erasure_coding"  # Tasks this worker can handle
+  maxConcurrent: 3  # Maximum concurrent tasks per worker
   
   # Working directory for task execution
   # Default: "/tmp/seaweedfs-worker"
@@ -249,14 +248,14 @@ worker:
       memory: "2Gi"
 ```
 
-### Worker Job Types
+### Worker Capabilities
 
-Workers can be configured with different job types:
+Workers can be configured with different capabilities:
 - **vacuum**: Reclaim deleted file space
-- **volume_balance**: Balance volumes across volume servers
+- **balance**: Balance volumes across volume servers
 - **erasure_coding**: Handle erasure coding operations
 
-You can configure workers with all job types or create specialized worker pools with specific job types.
+You can configure workers with all capabilities or create specialized worker pools with specific capabilities.
 
 ### Worker Deployment Strategy
 
@@ -265,11 +264,11 @@ For production deployments, consider:
 1. **Multiple Workers**: Deploy 2+ worker replicas for high availability
 2. **Resource Allocation**: Workers need sufficient CPU/memory for maintenance tasks
 3. **Storage**: Workers need temporary storage for vacuum and balance operations (size depends on volume size)
-4. **Specialized Workers**: Create separate worker deployments for different job types if needed
+4. **Specialized Workers**: Create separate worker deployments for different capabilities if needed
 
 Example specialized worker configuration:
 
-For specialized worker pools, deploy separate Helm releases with different job types:
+For specialized worker pools, deploy separate Helm releases with different capabilities:
 
 **values-worker-vacuum.yaml** (for vacuum operations):
 ```yaml
@@ -288,8 +287,8 @@ admin:
 worker:
   enabled: true
   replicas: 2
-  jobType: "vacuum"
-  maxExecute: 2
+  capabilities: "vacuum"
+  maxConcurrent: 2
   # REQUIRED: Point to the admin service of your main SeaweedFS release
   # Replace <namespace> with the namespace where your main seaweedfs is deployed
   # Example: If deploying in namespace "production":
@@ -314,8 +313,8 @@ admin:
 worker:
   enabled: true
   replicas: 1
-  jobType: "volume_balance"
-  maxExecute: 1
+  capabilities: "balance"
+  maxConcurrent: 1
   # REQUIRED: Point to the admin service of your main SeaweedFS release
   # Replace <namespace> with the namespace where your main seaweedfs is deployed
   # Example: If deploying in namespace "production":
@@ -324,7 +323,6 @@ worker:
 ```
 
 Deploy the specialized workers as separate releases:
-### Specialized Worker Deployment
 ```bash
 # Deploy vacuum workers
 helm install seaweedfs-worker-vacuum seaweedfs/seaweedfs -f values-worker-vacuum.yaml
@@ -333,22 +331,6 @@ helm install seaweedfs-worker-vacuum seaweedfs/seaweedfs -f values-worker-vacuum
 helm install seaweedfs-worker-balance seaweedfs/seaweedfs -f values-worker-balance.yaml
 ```
 
-## OpenShift Support
-
-SeaweedFS can be deployed on OpenShift or any cluster enforcing the Kubernetes "restricted" Pod Security Standard. By default, OpenShift blocks containers that run as root or use `hostPath` volumes.
-
-To deploy on OpenShift, use the provided `openshift-values.yaml` which overrides the default configuration to:
-1. Use `PersistentVolumeClaims` instead of `hostPath`.
-2. Enable `runAsNonRoot` and omit hardcoded UIDs to allow OpenShift to assign valid UIDs automatically.
-3. Apply appropriate `seccompProfile` and drop capabilities.
-
-Usage:
-```bash
-helm install seaweedfs seaweedfs/seaweedfs \
-  -n seaweedfs --create-namespace \
-  -f openshift-values.yaml
-```
-
 ## Enterprise
 
 For enterprise users, please visit [seaweedfs.com](https://seaweedfs.com) for the SeaweedFS Enterprise Edition, 

k8s/charts/seaweedfs/openshift-values.yaml

@@ -1,116 +0,0 @@
-# openshift-values.yaml
-#
-# Example overrides for deploying SeaweedFS on OpenShift (or any cluster
-# enforcing the Kubernetes "restricted" Pod Security Standard).
-#
-# OpenShift's default "restricted" SCC blocks containers that:
-#   - Run as UID 0 (root)
-#   - Request privilege escalation
-#   - Use hostPath volumes
-#   - Omit a seccompProfile
-#
-# These overrides satisfy all four requirements by:
-#   1. Replacing hostPath volumes with PersistentVolumeClaims (or emptyDir for logs)
-#   2. Enabling runAsNonRoot: true. By omitting runAsUser, OpenShift will
-#      automatically assign a valid UID from the namespace's allocated range.
-#   3. Dropping all Linux capabilities and setting allowPrivilegeEscalation: false
-#   4. Enabling RuntimeDefault seccompProfile
-#
-# Usage:
-#   helm install seaweedfs seaweedfs/seaweedfs \
-#     -n seaweedfs --create-namespace \
-#     -f openshift-values.yaml
-#
-# Adjust storageClass and sizes to match your cluster's available StorageClasses.
-# On OpenShift you can discover them with: oc get storageclass
-
-master:
-  data:
-    type: "persistentVolumeClaim"
-    size: "10Gi"
-    storageClass: ""        # leave empty to use the cluster default StorageClass
-
-  logs:
-    type: "emptyDir"        # avoids hostPath; use persistentVolumeClaim if you need log persistence
-
-  podSecurityContext:
-    enabled: true
-    # On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
-    # controller assign them automatically based on the namespace's SCC.
-    runAsNonRoot: true
-
-  containerSecurityContext:
-    enabled: true
-    allowPrivilegeEscalation: false
-    capabilities:
-      drop: ["ALL"]
-    runAsNonRoot: true
-    seccompProfile:
-      type: RuntimeDefault
-
-volume:
-  dataDirs:
-    - name: data1
-      type: "persistentVolumeClaim"
-      size: "100Gi"
-      storageClass: ""      # leave empty to use the cluster default StorageClass
-      maxVolumes: 0
-
-  logs:
-    type: "emptyDir"
-
-  podSecurityContext:
-    enabled: true
-    # On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
-    # controller assign them automatically based on the namespace's SCC.
-    runAsNonRoot: true
-
-  containerSecurityContext:
-    enabled: true
-    allowPrivilegeEscalation: false
-    capabilities:
-      drop: ["ALL"]
-    runAsNonRoot: true
-    seccompProfile:
-      type: RuntimeDefault
-
-filer:
-  data:
-    type: "persistentVolumeClaim"
-    size: "25Gi"
-    storageClass: ""        # leave empty to use the cluster default StorageClass
-
-  logs:
-    type: "emptyDir"
-
-  podSecurityContext:
-    enabled: true
-    # On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
-    # controller assign them automatically based on the namespace's SCC.
-    runAsNonRoot: true
-
-  containerSecurityContext:
-    enabled: true
-    allowPrivilegeEscalation: false
-    capabilities:
-      drop: ["ALL"]
-    runAsNonRoot: true
-    seccompProfile:
-      type: RuntimeDefault
-
-# S3 gateway (if enabled)
-s3:
-  podSecurityContext:
-    enabled: true
-    # On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
-    # controller assign them automatically based on the namespace's SCC.
-    runAsNonRoot: true
-
-  containerSecurityContext:
-    enabled: true
-    allowPrivilegeEscalation: false
-    capabilities:
-      drop: ["ALL"]
-    runAsNonRoot: true
-    seccompProfile:
-      type: RuntimeDefault

k8s/charts/seaweedfs/templates/s3/s3-secret.yaml

@@ -10,21 +10,10 @@
 {{- if and .Values.s3.reuseLegacySecret $existingSecret }}
 {{- $reuse = true }}
 {{- end }}
-{{- $creds := .Values.s3.credentials | default dict -}}
-{{- $adminCreds := $creds.admin | default dict -}}
-{{- $access_key_admin := $adminCreds.accessKey -}}
-{{- $secret_key_admin := $adminCreds.secretKey -}}
-{{- if not (and $access_key_admin $secret_key_admin) -}}
-  {{- $access_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
-  {{- $secret_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
-{{- end -}}
-{{- $readCreds := $creds.read | default dict -}}
-{{- $access_key_read := $readCreds.accessKey -}}
-{{- $secret_key_read := $readCreds.secretKey -}}
-{{- if not (and $access_key_read $secret_key_read) -}}
-  {{- $access_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
-  {{- $secret_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
-{{- end -}}
+{{- $access_key_admin := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
+{{- $secret_key_admin := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
+{{- $access_key_read := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
+{{- $secret_key_read := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
 apiVersion: v1
 kind: Secret
 type: Opaque
@@ -54,4 +43,4 @@ stringData:
   s3_auditLogConfig.json: |
     {{ toJson .Values.s3.auditLogConfig | nindent 4 }}
   {{- end }}
-{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/sftp/sftp-service.yaml

@@ -6,7 +6,6 @@ metadata:
   namespace: {{ .Release.Namespace }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/component: sftp
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}

k8s/charts/seaweedfs/templates/shared/_helpers.tpl

@@ -135,9 +135,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- $repositoryName := default .Values.image.repository .Values.global.repository | toString -}}
 {{- $name := .Values.global.imageName | toString -}}
 {{- $tag := default .Chart.AppVersion .Values.image.tag  | toString -}}
-{{- if .Values.image.repository -}}
-{{-   $name = $repositoryName -}}
-{{- else if $repositoryName -}}
+{{- if $repositoryName -}}
 {{-   $name = printf "%s/%s" (trimSuffix "/" $repositoryName) (base $name) -}}
 {{- end -}}
 {{- if $registryName -}}

k8s/charts/seaweedfs/templates/volume/volume-service.yaml

@@ -12,7 +12,6 @@ metadata:
   namespace: {{ $.Release.Namespace }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" $ }}
-    app.kubernetes.io/instance: {{ $.Release.Name }}
     app.kubernetes.io/component: {{ $volumeName }}
     helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: {{ $.Release.Service }}

k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml

@@ -136,15 +136,11 @@ spec:
               {{- else }}
               -admin={{ template "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }}{{ if .Values.admin.grpcPort }}.{{ .Values.admin.grpcPort }}{{ end }} \
               {{- end }}
-              -jobType={{ .Values.worker.jobType }} \
-              -maxDetect={{ .Values.worker.maxDetect }} \
-              -maxExecute={{ .Values.worker.maxExecute }} \
-              -workingDir={{ .Values.worker.workingDir }}{{- if or .Values.worker.metricsPort .Values.worker.metricsIp .Values.worker.extraArgs }} \{{ end }}
+              -capabilities={{ .Values.worker.capabilities }} \
+              -maxConcurrent={{ .Values.worker.maxConcurrent }} \
+              -workingDir={{ .Values.worker.workingDir }}{{- if or .Values.worker.metricsPort .Values.worker.extraArgs }} \{{ end }}
               {{- if .Values.worker.metricsPort }}
-              -metricsPort={{ .Values.worker.metricsPort }}{{- if or .Values.worker.metricsIp .Values.worker.extraArgs }} \{{ end }}
-              {{- end }}
-              {{- if .Values.worker.metricsIp }}
-              -metricsIp={{ .Values.worker.metricsIp }}{{- if .Values.worker.extraArgs }} \{{ end }}
+              -metricsPort={{ .Values.worker.metricsPort }}{{- if .Values.worker.extraArgs }} \{{ end }}
               {{- end }}
               {{- range $index, $arg := .Values.worker.extraArgs }}
               {{ $arg }}{{- if lt $index (sub (len $.Values.worker.extraArgs) 1) }} \{{ end }}

k8s/charts/seaweedfs/values.yaml

@@ -890,10 +890,6 @@ filer:
     # set to the name of an existing kubernetes Secret with the s3 json config file
     # should have a secret key called seaweedfs_s3_config with an inline json configure
     existingConfigSecret: null
-    # To provide explicit credentials for the S3 gateway, set them under
-    # the top-level s3.credentials key (not filer.s3.credentials).
-    # The s3-secret.yaml template only reads from .Values.s3.credentials.
-    # See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey
     auditLogConfig: {}
     # You may specify buckets to be created during the install or upgrade process.
     # Buckets may be exposed publicly by setting `anonymousRead` to `true`
@@ -922,16 +918,6 @@ s3:
   # set to the name of an existing kubernetes Secret with the s3 json config file
   # should have a secret key called seaweedfs_s3_config with an inline json config
   existingConfigSecret: null
-  # Optionally provide explicit credentials for the S3 gateway.
-  # When set, these are used in the generated s3 secret instead of
-  # auto-generating random credentials.
-  # credentials:
-  #   admin:
-  #     accessKey: ""
-  #     secretKey: ""
-  #   read:
-  #     accessKey: ""
-  #     secretKey: ""
   auditLogConfig: {}
   # You may specify buckets to be created during the install or upgrade process.
   # Buckets may be exposed publicly by setting `anonymousRead` to `true`
@@ -1284,20 +1270,17 @@ worker:
   replicas: 1
   loggingOverrideLevel: null
   metricsPort: 9327
-  metricsIp: ""  # If empty, defaults to 0.0.0.0
 
   # Admin server to connect to
   adminServer: ""
 
-  # Worker job types - comma-separated list
-  # Available: vacuum, volume_balance, erasure_coding
-  jobType: "vacuum,volume_balance,erasure_coding"
+  # Worker capabilities - comma-separated list
+  # Available: vacuum, balance, erasure_coding
+  # Default: "vacuum,balance,erasure_coding" (all capabilities)
+  capabilities: "vacuum,balance,erasure_coding"
 
-  # Maximum number of concurrent detection requests
-  maxDetect: 1
-
-  # Maximum number of concurrent execution jobs
-  maxExecute: 4
+  # Maximum number of concurrent tasks
+  maxConcurrent: 3
 
   # Working directory for task execution
   workingDir: "/tmp/seaweedfs-worker"
@@ -1435,10 +1418,6 @@ allInOne:
     # Set to the name of an existing kubernetes Secret with the s3 json config file
     # should have a secret key called seaweedfs_s3_config with an inline json config
     existingConfigSecret: null
-    # To provide explicit credentials for the S3 gateway, set them under
-    # the top-level s3.credentials key (not allInOne.s3.credentials).
-    # The s3-secret.yaml template only reads from .Values.s3.credentials.
-    # See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey
     auditLogConfig: null  # S3 audit log configuration (null inherits from s3.auditLogConfig)
     # You may specify buckets to be created during the install process.
     # Buckets may be exposed publicly by setting `anonymousRead` to `true`

rust/volume_server/Cargo.lock

@@ -0,0 +1,7 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "weed-volume-rs"
+version = "0.1.0"

rust/volume_server/Cargo.toml

@@ -0,0 +1,6 @@
+[package]
+name = "weed-volume-rs"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]

rust/volume_server/DEV_PLAN.md

@@ -0,0 +1,245 @@
+# Rust Volume Server Parity Implementation Plan
+
+## Objective
+Implement a native Rust volume server that replicates Go volume-server behavior for HTTP and gRPC APIs, so it can become a drop-in replacement validated by existing integration suites.
+
+## Current Focus (2026-02-16)
+- Program focus is now Rust implementation parity, not broad test expansion.
+- `test/volume_server` is treated as the parity gate.
+- Existing Rust launcher modes (`exec`, `proxy`) are transition tools; they are not the final target.
+
+## Current Status
+- Rust crate and launcher are in place.
+- Integration harness can run:
+  - Go master + Go volume (default)
+  - Go master + Rust launcher (`VOLUME_SERVER_IMPL=rust`)
+- Rust launcher `proxy` mode has full-suite integration pass while delegating backend handlers to Go.
+- Rust launcher `native` mode is wired as the default Rust entrypoint and currently bootstraps via supervised Go backend delegation.
+- Native Rust HTTP handlers now serve control/surface paths in `native` mode:
+  - `/status`, `/healthz`
+  - `OPTIONS` admin/public behavior (method allow-list + CORS preflight headers)
+  - `/ui/index.html` with config-driven JWT/access-ui gating
+  - `/favicon.ico` and `/seaweedfsstatic/*`
+  - public non-read methods (`POST`/`PUT`/`DELETE`/unsupported verbs) as `200` no-op passthrough parity
+  - admin unsupported verbs as `400` parity
+- Native Rust control-surface parity now also includes:
+  - `/healthz` status mirroring from backend state transitions (e.g. leave/stopping => `503`)
+  - absolute-form HTTP request-target normalization before native route matching
+- Native Rust HTTP data-path prevalidation now includes:
+  - early malformed vid/fid rejection (`400`) for GET/HEAD/POST/PUT fid-route shapes before delegation
+  - slash-form fid-route parsing for `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` with reserved-path exclusions (`/status`, `/healthz`, `/ui/index.html`, `/stats/*`, static assets)
+  - write-path error prevalidation for fid routes:
+    - malformed multipart form-data requests without a boundary => `400`
+    - requests containing `Content-MD5` header => `400` mismatch parity branch
+    - request bodies exceeding configured `-fileSizeLimitMB` (by `Content-Length`) => `400`
+- Native Rust API/storage handlers are not implemented yet.
+
+## Parity Exit Criteria
+1. Native mode passes:
+   - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
+   - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
+2. CI runs native Rust mode integration coverage (at least smoke, then expanded shards).
+3. Rust mode defaults to native behavior for integration harness.
+4. Go-backend delegation is removed (or retained only as explicit fallback mode).
+
+## Architecture Workstreams
+
+### A. Runtime and Configuration Parity
+- [x] Add `native` runtime mode in `weed-volume-rs` (bootstrap delegation path).
+- [ ] Parse and honor volume-server CLI/config flags used by integration harness:
+  - [ ] network/bind ports (`-ip`, `-port`, `-port.grpc`, `-port.public`)
+  - [ ] master target/config dir/read mode/throttling/JWT-related config
+  - [ ] size/timeout controls and maintenance state defaults
+- [ ] Implement graceful lifecycle behavior (signals, shutdown, readiness).
+
+### B. Native HTTP Surface
+- [ ] Admin/control endpoints:
+  - [x] `GET /status` (native Rust in `native` mode)
+  - [x] `GET /healthz` (native Rust in `native` mode)
+  - [x] `OPTIONS` admin/public method+CORS control behavior (native Rust in `native` mode)
+  - [x] static/UI endpoints currently exercised
+- [ ] Data read path parity:
+  - [ ] fid parsing/path variants
+  - [ ] conditional headers (`If-Modified-Since`, `If-None-Match`)
+  - [ ] range handling (single/multi/invalid)
+  - [ ] deleted reads, auth checks, read-mode branches
+  - [ ] chunk-manifest and compression/image transformation branches
+- [ ] Data write/delete parity:
+  - [ ] write success/unchanged/error paths
+  - [ ] replication and file-size-limit paths
+  - [ ] delete and chunk-manifest delete branches
+- [ ] Method/CORS/public-port parity for split admin/public behavior.
+
+### C. Native gRPC Surface
+- [ ] Control-plane RPCs:
+  - [ ] `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`
+  - [ ] admin lifecycle: allocate/mount/unmount/delete/configure/readonly/writable
+- [ ] Data RPCs:
+  - [ ] `ReadNeedleBlob`, `ReadNeedleMeta`, `WriteNeedleBlob`
+  - [ ] `BatchDelete`, `ReadAllNeedles`
+  - [ ] sync/copy/receive and status endpoints
+- [ ] Stream RPCs:
+  - [ ] tail sender/receiver
+  - [ ] vacuum streams
+  - [ ] query streams
+- [ ] Advanced families:
+  - [ ] erasure coding RPC set
+  - [ ] tiering/remote fetch
+  - [ ] scrub/query mode matrix
+
+### D. Storage Compatibility Layer
+- [ ] Implement volume data/index handling compatible with Go on-disk format.
+- [ ] Preserve cookie/checksum/timestamp semantics used by tests.
+- [ ] Match read/write/delete consistency and error mapping behavior.
+- [ ] Ensure EC metadata/data-path compatibility with existing files.
+
+### E. Operational Hardening
+- [ ] Deterministic startup/readiness and shutdown semantics.
+- [ ] Log/error parity sufficient for debugging and CI triage.
+- [ ] Concurrency/timeout behavior alignment for throttling and streams.
+- [ ] Performance baseline checks vs Go for key flows.
+
+## Milestone Plan
+
+### M0 (Completed): Harness + Launcher Transition
+- [x] Rust launcher integrated into harness.
+- [x] Proxy mode full-suite validation with Go backend delegation.
+
+### M1: Native Skeleton (Control Plane First)
+- [ ] `native` mode boots and serves:
+  - [x] `/status`, `/healthz`
+  - [ ] `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`
+- Gate:
+  - [x] targeted HTTP/grpc control tests pass in `native` mode (delegated backend path).
+
+### M2: Native Core Data Paths
+- [ ] Native HTTP read/write/delete baseline parity.
+- [ ] Native gRPC data baseline parity (`Read/WriteNeedle*`, `BatchDelete`, `ReadAllNeedles`).
+- Gate:
+  - core HTTP and gRPC data suites pass in `native` mode.
+
+### M3: Native Stream + Copy/Sync
+- [ ] Tail/copy/receive/sync paths in native mode.
+- Gate:
+  - stream/copy families pass in `native` mode.
+
+### M4: Native Advanced Feature Families
+- [ ] EC, tiering, scrub/query advanced branches.
+- Gate:
+  - full `/test/volume_server/http` and `/test/volume_server/grpc` pass in `native` mode.
+
+### M5: CI/Cutover
+- [x] Add/expand native-mode CI jobs (smoke matrix includes `native`).
+- [x] Make native mode default for Rust integration runs.
+- [ ] Keep `exec`/`proxy` only as explicit fallback modes during rollout.
+
+## Immediate Next Steps
+1. Implement minimal native gRPC state/ping RPC handlers (`GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`).
+2. Expand native HTTP parity from control/static surface into data-path handlers (read/write/delete variants) while preserving current native control behavior.
+3. Keep rerunning native-mode integration suites as each delegated branch is replaced.
+4. Add mismatch triage notes for each API moved from delegation to native implementation.
+
+## Risk Register
+- On-disk format mismatch risk:
+  - Mitigation: implement format-level compatibility tests early (idx/dat/needle encoding).
+- Behavioral drift in edge branches:
+  - Mitigation: use integration suite failures as primary truth; only add tests for newly discovered untracked branches.
+- Stream/concurrency semantic mismatch:
+  - Mitigation: stabilize with focused interruption/timeout parity tests.
+
+## Progress Log
+- Date: 2026-02-15
+- Change: Added Rust launcher integration (`exec`) and harness wiring.
+- Validation: Rust launcher mode passed smoke and full integration suites while delegating to Go backend.
+- Commits: `7beab85c2`, `880c2e1da`, `63d08e8a9`, `d402573ea`, `3bd20e6a1`, `6ce4d7ede`
+
+- Date: 2026-02-15
+- Change: Added Rust proxy supervisor mode and validated full integration suite.
+- Validation:
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=proxy go test -count=1 ./test/volume_server/http`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=proxy go test -count=1 ./test/volume_server/grpc`
+- Commits: `a7f50d23b`, `548b3d9a3`
+
+- Date: 2026-02-16
+- Change: Re-focused plan from test expansion to native Rust implementation parity.
+- Validation basis: latest Rust proxy full-suite pass keeps regression baseline stable while native implementation starts.
+- Commits: `14c863dbf`
+
+- Date: 2026-02-16
+- Change: Added native Rust launcher mode bootstrap, set Rust launcher default mode to `native`, and expanded CI Rust smoke matrix to include `native`.
+- Validation:
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
+  - `env VOLUME_SERVER_IMPL=rust go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
+  - `env VOLUME_SERVER_IMPL=rust go test -count=1 ./test/volume_server/grpc -run '^TestStateAndStatusRPCs$'`
+- Commits: `70ddbee37`, `61befd10f`, `2e65966c0`
+
+- Date: 2026-02-16
+- Change: Implemented first native Rust HTTP handlers in `native` mode for `/status` and `/healthz` on the admin listener while preserving proxy delegation for other APIs.
+- Validation:
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
+- Commits: `7e6e0261a`
+
+- Date: 2026-02-16
+- Change: Implemented native Rust `OPTIONS` handling for admin and public listeners in `native` mode, including method allow-list and origin-driven CORS response parity used by integration tests.
+- Validation:
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestOptionsMethodsByPort$|^TestOptionsWithOriginIncludesCorsHeaders$'`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
+- Commits: `fbff2cb39`
+
+- Date: 2026-02-16
+- Change: Added native Rust UI/static/public-surface handlers in `native` mode:
+  - `/ui/index.html` with config-driven JWT/access-ui gating parity
+  - `/favicon.ico` and `/seaweedfsstatic/*` static endpoint parity
+  - public non-read methods as `200` no-op passthrough parity
+  - admin unsupported methods as `400` parity
+- Validation:
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$|^TestUiIndexNotExposedWhenJwtSigningEnabled$|^TestUiIndexExposedWhenJwtSigningEnabledAndAccessUITrue$|^TestStaticAssetEndpoints$|^TestStaticAssetEndpointsOnPublicPort$'`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestOptionsMethodsByPort$|^TestOptionsWithOriginIncludesCorsHeaders$|^TestPublicPortReadOnlyMethodBehavior$|^TestCorsAndUnsupportedMethodBehavior$|^TestUnsupportedMethodTraceParity$|^TestUnsupportedMethodPropfindParity$|^TestUnsupportedMethodConnectParity$|^TestUnsupportedMethodMkcolParity$'`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
+- Commits: `23e4497b2`
+
+- Date: 2026-02-16
+- Change: Hardened native Rust HTTP control-path parity:
+  - `/healthz` now mirrors backend health status (`200`/`503`) in native mode
+  - absolute-form request-target parsing (e.g. `GET http://host/path HTTP/1.1`) now normalizes to route path parity
+- Validation:
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
+- Commits: `d6ff6ed6d`
+
+- Date: 2026-02-16
+- Change: Added native malformed fid-route validation in `native` mode:
+  - GET/HEAD/POST/PUT requests matching fid URL shapes now return Rust-native `400 Bad Request` for invalid vid/fid tokens
+  - valid fid-route requests still delegate to backend handlers for storage/data execution paths
+- Validation:
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestInvalidReadPathReturnsBadRequest$|^TestWriteInvalidVidAndFidReturnBadRequest$|^TestMalformedVidFidPathReturnsBadRequest$|^TestDownloadLimitInvalidVidWhileOverLimitReturnsBadRequest$'`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
+- Commits: `1ce0174b2`
+
+- Date: 2026-02-16
+- Change: Added native Rust write-path prevalidation branches for fid routes in `native` mode:
+  - multipart form uploads without boundary now return native `400`
+  - `Content-MD5` requests now return native `400` mismatch branch
+  - `Content-Length` over configured `-fileSizeLimitMB` now returns native `400` limit branch
+- Validation:
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestUploadReadRangeHeadDeleteRoundTrip$|^TestWriteMalformedMultipartAndMD5Mismatch$|^TestWriteRejectsPayloadOverFileSizeLimit$'`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
+- Commits: `94cefd6f4`
+
+- Date: 2026-02-16
+- Change: Broadened native fid-route parser to cover slash-form URLs:
+  - `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` now flow through native malformed vid/fid validation
+  - added explicit non-fid exclusions for control/stats/static paths to preserve route parity
+- Validation:
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$|^TestReadPathShapesAndIfModifiedSince$|^TestMalformedVidFidPathReturnsBadRequest$'`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
+  - `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
+- Commits: `e66983e9b`

test/erasure_coding/admin_dockertest/ec_integration_test.go

@@ -2,11 +2,13 @@ package admin_dockertest
 
 import (
 	"bytes"
-	crand "crypto/rand"
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"math/rand"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -159,95 +161,129 @@ func TestEcEndToEnd(t *testing.T) {
 
 	client := &http.Client{}
 
-	// 1. Configure plugin job types for fast EC detection/execution.
-	t.Log("Configuring plugin job types via API...")
+	// 1. Configure Global Maintenance (Scan Interval = 1s) via API
+	t.Log("Configuring Global Maintenance via API...")
 
-	// Disable volume balance to reduce interference for this EC-focused test.
-	balanceConfig := map[string]interface{}{
-		"job_type": "volume_balance",
-		"admin_runtime": map[string]interface{}{
-			"enabled": false,
-		},
-	}
-	jsonBody, err := json.Marshal(balanceConfig)
-	if err != nil {
-		t.Fatalf("Failed to marshal volume_balance config: %v", err)
-	}
-	req, err := http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/volume_balance/config", bytes.NewBuffer(jsonBody))
-	if err != nil {
-		t.Fatalf("Failed to create volume_balance config request: %v", err)
-	}
-	req.Header.Set("Content-Type", "application/json")
+	// 1.1 Fetch current config
+	req, _ := http.NewRequest("GET", AdminUrl+"/api/maintenance/config", nil)
 	resp, err := client.Do(req)
 	if err != nil {
-		t.Fatalf("Failed to update volume_balance config: %v", err)
+		t.Fatalf("Failed to get global config: %v", err)
 	}
 	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
-		t.Fatalf("Failed to update volume_balance config (status %d): %s", resp.StatusCode, string(body))
+		t.Fatalf("Failed to get global config (status %d): %s", resp.StatusCode, string(body))
+	}
+
+	var globalConfig map[string]interface{}
+	if err := json.NewDecoder(resp.Body).Decode(&globalConfig); err != nil {
+		t.Fatalf("Failed to decode global config: %v", err)
 	}
 	resp.Body.Close()
 
-	ecConfig := map[string]interface{}{
-		"job_type": "erasure_coding",
-		"admin_runtime": map[string]interface{}{
-			"enabled":                          true,
-			"detection_interval_seconds":       1,
-			"global_execution_concurrency":     4,
-			"per_worker_execution_concurrency": 4,
-			"max_jobs_per_detection":           100,
-		},
-		"worker_config_values": map[string]interface{}{
-			"quiet_for_seconds": map[string]interface{}{
-				"int64_value": "1",
-			},
-			"min_interval_seconds": map[string]interface{}{
-				"int64_value": "1",
-			},
-			"min_size_mb": map[string]interface{}{
-				"int64_value": "1",
-			},
-			"fullness_ratio": map[string]interface{}{
-				"double_value": 0.0001,
-			},
-		},
+	// 1.2 Modify config
+	globalConfig["enabled"] = true
+	globalConfig["scan_interval_seconds"] = 1
+
+	// Ensure policy structure exists
+	if globalConfig["policy"] == nil {
+		globalConfig["policy"] = map[string]interface{}{}
 	}
-	jsonBody, err = json.Marshal(ecConfig)
-	if err != nil {
-		t.Fatalf("Failed to marshal erasure_coding config: %v", err)
+	policy, _ := globalConfig["policy"].(map[string]interface{})
+
+	// Ensure task_policies structure exists
+	if policy["task_policies"] == nil {
+		policy["task_policies"] = map[string]interface{}{}
 	}
-	req, err = http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/erasure_coding/config", bytes.NewBuffer(jsonBody))
-	if err != nil {
-		t.Fatalf("Failed to create erasure_coding config request: %v", err)
+	taskPolicies, _ := policy["task_policies"].(map[string]interface{})
+
+	// Disable balance tasks to avoid interference with EC test
+	if taskPolicies["balance"] == nil {
+		taskPolicies["balance"] = map[string]interface{}{}
 	}
+	balancePolicy, _ := taskPolicies["balance"].(map[string]interface{})
+	balancePolicy["enabled"] = false
+
+	// Set global max concurrent
+	policy["global_max_concurrent"] = 4
+	globalConfig["policy"] = policy
+
+	// Explicitly set required fields
+	requiredFields := map[string]float64{
+		"worker_timeout_seconds":   300,
+		"task_timeout_seconds":     7200,
+		"retry_delay_seconds":      900,
+		"cleanup_interval_seconds": 86400,
+		"task_retention_seconds":   604800,
+		"max_retries":              3,
+	}
+	for field, val := range requiredFields {
+		if _, ok := globalConfig[field]; !ok || globalConfig[field] == 0 {
+			globalConfig[field] = val
+		}
+	}
+
+	// 1.3 Update config
+	jsonBody, _ := json.Marshal(globalConfig)
+	req, _ = http.NewRequest("PUT", AdminUrl+"/api/maintenance/config", bytes.NewBuffer(jsonBody))
 	req.Header.Set("Content-Type", "application/json")
 	resp, err = client.Do(req)
 	if err != nil {
-		t.Fatalf("Failed to update erasure_coding config: %v", err)
+		t.Fatalf("Failed to update global config: %v", err)
 	}
 	if resp.StatusCode != http.StatusOK {
 		body, _ := io.ReadAll(resp.Body)
-		t.Fatalf("Failed to update erasure_coding config (status %d): %s", resp.StatusCode, string(body))
+		t.Fatalf("Failed to update global config (status %d): %s", resp.StatusCode, string(body))
 	}
 	resp.Body.Close()
 
-	// 2. Upload a file
+	// 2. Configure EC Task (Short intervals) via Form API
+	t.Log("Configuring EC Task via Form API...")
+	formData := url.Values{}
+	formData.Set("enabled", "true")
+	formData.Set("scan_interval_seconds", "1")
+	formData.Set("repeat_interval_seconds", "1")
+	formData.Set("check_interval_seconds", "1")
+	formData.Set("max_concurrent", "4")
+	formData.Set("quiet_for_seconds_value", "1")
+	formData.Set("quiet_for_seconds_unit", "seconds")
+	formData.Set("min_size_mb", "1")
+	formData.Set("fullness_ratio", "0.0001")
+
+	req, _ = http.NewRequest("POST", AdminUrl+"/maintenance/config/erasure_coding", strings.NewReader(formData.Encode()))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	resp, err = client.Do(req)
+	if err != nil {
+		t.Fatalf("Failed to update EC config: %v", err)
+	}
+	if resp.StatusCode != 200 && resp.StatusCode != 303 {
+		body, _ := io.ReadAll(resp.Body)
+		t.Fatalf("Failed to update EC config (status %d): %s", resp.StatusCode, string(body))
+	}
+	resp.Body.Close()
+	t.Log("EC Task Configuration updated")
+
+	// 3. Restart Admin to pick up Global Config (Scan Interval)
+	if len(runningCmds) > 0 {
+		adminCmd := runningCmds[len(runningCmds)-1]
+		t.Log("Restarting Admin Server to apply configuration...")
+		stopWeed(t, adminCmd)
+		time.Sleep(10 * time.Second)
+		startWeed(t, "admin_restarted", "admin", "-master=localhost:9333", "-port=23646", "-port.grpc=33646", "-dataDir=./tmp/admin")
+		waitForUrl(t, AdminUrl+"/health", 60)
+	}
+
+	// 4. Upload a file
 	fileSize := 5 * 1024 * 1024
 	data := make([]byte, fileSize)
-	crand.Read(data)
+	rand.Read(data)
 	fileName := fmt.Sprintf("ec_test_file_%d", time.Now().Unix())
 	t.Logf("Uploading %d bytes file %s to Filer...", fileSize, fileName)
 	uploadUrl := FilerUrl + "/" + fileName
 
 	var uploadErr error
 	for i := 0; i < 10; i++ {
-		req, err := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(data))
-		if err != nil {
-			uploadErr = err
-			t.Logf("Upload attempt %d failed to create request: %v", i+1, err)
-			time.Sleep(2 * time.Second)
-			continue
-		}
+		req, _ := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(data))
 		resp, err := client.Do(req)
 		if err == nil {
 			if resp.StatusCode == 201 {
@@ -270,17 +306,17 @@ func TestEcEndToEnd(t *testing.T) {
 	}
 	t.Log("Upload successful")
 
-	// 3. Verify EC Encoding
+	// 5. Verify EC Encoding
 	t.Log("Waiting for EC encoding (checking Master topology)...")
 	startTime := time.Now()
 	ecVerified := false
 	var lastBody []byte
 
 	for time.Since(startTime) < 300*time.Second {
-		// 3.1 Check Master Topology
+		// 5.1 Check Master Topology
 		resp, err := http.Get(MasterUrl + "/dir/status")
 		if err == nil {
-			lastBody, _ = io.ReadAll(resp.Body)
+			lastBody, _ = ioutil.ReadAll(resp.Body)
 			resp.Body.Close()
 
 			// Check total EC shards
@@ -300,8 +336,8 @@ func TestEcEndToEnd(t *testing.T) {
 			}
 		}
 
-		// 3.2 Debug: Check workers and jobs
-		wResp, wErr := http.Get(AdminUrl + "/api/plugin/workers")
+		// 5.2 Debug: Check workers and tasks
+		wResp, wErr := http.Get(AdminUrl + "/api/maintenance/workers")
 		workerCount := 0
 		if wErr == nil {
 			var workers []interface{}
@@ -310,7 +346,7 @@ func TestEcEndToEnd(t *testing.T) {
 			workerCount = len(workers)
 		}
 
-		tResp, tErr := http.Get(AdminUrl + "/api/plugin/jobs?limit=1000")
+		tResp, tErr := http.Get(AdminUrl + "/api/maintenance/tasks")
 		taskCount := 0
 		if tErr == nil {
 			var tasks []interface{}

test/foundationdb/foundationdb_concurrent_test.go

@@ -1,4 +1,5 @@
 //go:build foundationdb
+// +build foundationdb
 
 package foundationdb
 

test/foundationdb/foundationdb_integration_test.go

@@ -1,4 +1,5 @@
 //go:build foundationdb
+// +build foundationdb
 
 package foundationdb
 

test/java/spark/docker-compose.yml

@@ -19,6 +19,9 @@ services:
       start_period: 10s
 
   seaweedfs-volume:
+    build:
+      context: ../../../docker
+      dockerfile: Dockerfile.local
     image: seaweedfs:local
     container_name: seaweedfs-spark-volume
     ports:
@@ -40,6 +43,9 @@ services:
       start_period: 10s
 
   seaweedfs-filer:
+    build:
+      context: ../../../docker
+      dockerfile: Dockerfile.local
     image: seaweedfs:local
     container_name: seaweedfs-spark-filer
     ports:

test/java/spark/pom.xml

@@ -116,7 +116,7 @@
             <dependency>
                 <groupId>org.apache.avro</groupId>
                 <artifactId>avro</artifactId>
-                <version>1.11.5</version>
+                <version>1.11.4</version>
             </dependency>
 
             <!-- Apache ZooKeeper - Fix CVEs -->

test/s3/cors/Makefile

@@ -12,7 +12,6 @@ FILER_PORT := 8888
 TEST_TIMEOUT := 10m
 TEST_PATTERN := TestCORS
 SERVER_DIR := test-mini-data
-S3_CONFIG := s3_test_config.json
 
 # Default target
 help:
@@ -81,15 +80,13 @@ start-server: check-deps
 	@echo "🔍 DEBUG: Creating volume directory..."
 	@mkdir -p $(SERVER_DIR)
 	@echo "🔍 DEBUG: Launching SeaweedFS S3 server in background..."
-	@echo "🔍 DEBUG: Command: AWS_ACCESS_KEY_ID=some_access_key1 AWS_SECRET_ACCESS_KEY=some_secret_key1 $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
-	@env AWS_ACCESS_KEY_ID=some_access_key1 \
-	 AWS_SECRET_ACCESS_KEY=some_secret_key1 \
-	 $(WEED_BINARY) mini \
+	@echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
+	@$(WEED_BINARY) mini \
 		-dir=$(SERVER_DIR) \
 		-s3.port=$(S3_PORT) \
 		-s3.config=$(S3_CONFIG) \
 		> weed-test.log 2>&1 & \
-		echo $$! > weed-server.pid
+		echo $$! > weed-test.pid
 	
 	@echo "Waiting for S3 server to be ready..."
 	@for i in $$(seq 1 30); do \
@@ -100,7 +97,7 @@ start-server: check-deps
 		sleep 1; \
 	done; \
 	echo "S3 server failed to start"; \
-	exit 1
+	exit 1 > weed-server.pid
 	@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
 	@echo "🔍 DEBUG: Checking if PID is still running..."
 	@sleep 2

test/s3/cors/s3_test_config.json

@@ -1,27 +0,0 @@
-{
-    "identities": [
-        {
-            "name": "anonymous",
-            "actions": [
-                "Read",
-                "List"
-            ]
-        },
-        {
-            "name": "admin",
-            "credentials": [
-                {
-                    "accessKey": "some_access_key1",
-                    "secretKey": "some_secret_key1"
-                }
-            ],
-            "actions": [
-                "Admin",
-                "Read",
-                "List",
-                "Tagging",
-                "Write"
-            ]
-        }
-    ]
-}

test/s3/iam/s3_iam_admin_test.go

@@ -4,7 +4,6 @@ import (
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -203,116 +202,6 @@ func TestIAMPolicyManagement(t *testing.T) {
 		})
 	})
 
-	t.Run("managed_policy_crud_lifecycle", func(t *testing.T) {
-		policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::*"}]}`
-
-		policyNames := []string{"test-managed-policy-lifecycle-a", "test-managed-policy-lifecycle-b"}
-		policyArns := make([]*string, 0, len(policyNames))
-		for _, policyName := range policyNames {
-			createResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
-				PolicyName:     aws.String(policyName),
-				PolicyDocument: aws.String(policyDoc),
-			})
-			require.NoError(t, err)
-			policyArns = append(policyArns, createResp.Policy.Arn)
-		}
-
-		t.Cleanup(func() {
-			for _, policyArn := range policyArns {
-				_, _ = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArn})
-			}
-		})
-
-		listResp, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
-		require.NoError(t, err)
-
-		foundByName := map[string]bool{}
-		for _, policy := range listResp.Policies {
-			if policy.PolicyName != nil {
-				foundByName[*policy.PolicyName] = true
-			}
-		}
-		for _, policyName := range policyNames {
-			assert.True(t, foundByName[policyName], "policy %s should be listed", policyName)
-		}
-
-		getResp, err := iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: policyArns[0]})
-		require.NoError(t, err)
-		require.NotNil(t, getResp.Policy)
-		assert.Equal(t, policyNames[0], aws.StringValue(getResp.Policy.PolicyName))
-		assert.Equal(t, aws.StringValue(policyArns[0]), aws.StringValue(getResp.Policy.Arn))
-
-		_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArns[0]})
-		require.NoError(t, err)
-
-		_, err = iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: policyArns[0]})
-		require.Error(t, err)
-		awsErr, ok := err.(awserr.Error)
-		require.True(t, ok)
-		assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
-
-		listAfterDeleteResp, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
-		require.NoError(t, err)
-		deletedPolicyFound := false
-		remainingPolicyFound := false
-		for _, policy := range listAfterDeleteResp.Policies {
-			if policy.PolicyName == nil {
-				continue
-			}
-			if *policy.PolicyName == policyNames[0] {
-				deletedPolicyFound = true
-			}
-			if *policy.PolicyName == policyNames[1] {
-				remainingPolicyFound = true
-			}
-		}
-		assert.False(t, deletedPolicyFound, "deleted policy should no longer be listed")
-		assert.True(t, remainingPolicyFound, "remaining policy should still be listed")
-
-		policyArns[0] = nil
-	})
-
-	t.Run("managed_policy_versions", func(t *testing.T) {
-		policyName := "test-managed-policy-version"
-		policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListBucket","Resource":"*"}]}`
-
-		createResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
-			PolicyName:     aws.String(policyName),
-			PolicyDocument: aws.String(policyDoc),
-		})
-		require.NoError(t, err)
-
-		t.Cleanup(func() {
-			_, _ = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: createResp.Policy.Arn})
-		})
-
-		listVersionsResp, err := iamClient.ListPolicyVersions(&iam.ListPolicyVersionsInput{
-			PolicyArn: createResp.Policy.Arn,
-		})
-		require.NoError(t, err)
-		require.NotEmpty(t, listVersionsResp.Versions)
-		assert.Equal(t, "v1", aws.StringValue(listVersionsResp.Versions[0].VersionId))
-		assert.Equal(t, true, aws.BoolValue(listVersionsResp.Versions[0].IsDefaultVersion))
-
-		getVersionResp, err := iamClient.GetPolicyVersion(&iam.GetPolicyVersionInput{
-			PolicyArn:  createResp.Policy.Arn,
-			VersionId:  aws.String("v1"),
-		})
-		require.NoError(t, err)
-		require.NotNil(t, getVersionResp.PolicyVersion)
-		assert.Equal(t, "v1", aws.StringValue(getVersionResp.PolicyVersion.VersionId))
-		assert.Contains(t, aws.StringValue(getVersionResp.PolicyVersion.Document), "s3:ListBucket")
-
-		_, err = iamClient.GetPolicyVersion(&iam.GetPolicyVersionInput{
-			PolicyArn: createResp.Policy.Arn,
-			VersionId: aws.String("v2"),
-		})
-		require.Error(t, err)
-		awsErr, ok := err.(awserr.Error)
-		require.True(t, ok)
-		assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
-	})
-
 	t.Run("user_inline_policy", func(t *testing.T) {
 		userName := "test-user-policy"
 		_, err := iamClient.CreateUser(&iam.CreateUserInput{

test/s3/normal/iam_test.go

@@ -1,6 +1,7 @@
 package example
 
 import (
+	"os"
 	"testing"
 	"time"
 
@@ -21,8 +22,10 @@ func TestIAMOperations(t *testing.T) {
 	// Set credentials before starting cluster
 	accessKey := "testkey123"
 	secretKey := "testsecret456"
-	t.Setenv("AWS_ACCESS_KEY_ID", accessKey)
-	t.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
+	os.Setenv("AWS_ACCESS_KEY_ID", accessKey)
+	os.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
+	defer os.Unsetenv("AWS_ACCESS_KEY_ID")
+	defer os.Unsetenv("AWS_SECRET_ACCESS_KEY")
 
 	// Create and start test cluster
 	cluster, err := startMiniCluster(t)

test/s3/normal/s3_integration_test.go

@@ -3,8 +3,6 @@ package example
 import (
 	"bytes"
 	"context"
-	"crypto/md5"
-	"encoding/base64"
 	"fmt"
 	"math/rand"
 	"net"
@@ -17,7 +15,6 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
@@ -71,22 +68,6 @@ func TestS3Integration(t *testing.T) {
 		testPutObject(t, cluster)
 	})
 
-	t.Run("UploadPart", func(t *testing.T) {
-		testPutPartWithChecksum(t, cluster)
-	})
-
-	t.Run("PutObjectWithChecksum", func(t *testing.T) {
-		testPutObjectWithChecksum(t, cluster)
-	})
-
-	t.Run("UploadPartWithChecksum", func(t *testing.T) {
-		testUploadPartWithChecksum(t, cluster)
-	})
-
-	t.Run("PutObjectWithChecksumAndSSEC", func(t *testing.T) {
-		testPutObjectWithChecksumAndSSEC(t, cluster)
-	})
-
 	t.Run("GetObject", func(t *testing.T) {
 		testGetObject(t, cluster)
 	})
@@ -116,9 +97,8 @@ func findAvailablePort() (int, error) {
 	return addr.Port, nil
 }
 
-// startMiniCluster starts a weed mini instance directly without exec.
-// Extra flags (e.g. "-s3.allowDeleteBucketNotEmpty=false") can be appended via extraArgs.
-func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
+// startMiniCluster starts a weed mini instance directly without exec
+func startMiniCluster(t *testing.T) (*TestCluster, error) {
 	// Find available ports
 	masterPort, err := findAvailablePort()
 	if err != nil {
@@ -166,14 +146,6 @@ func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
 		return nil, fmt.Errorf("failed to create security.toml: %v", err)
 	}
 
-	// Set environment variables for admin credentials safely for this test
-	if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
-		t.Setenv("AWS_ACCESS_KEY_ID", "admin")
-	}
-	if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
-		t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
-	}
-
 	// Start weed mini in a goroutine by calling the command directly
 	cluster.wg.Add(1)
 	go func() {
@@ -193,7 +165,7 @@ func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
 		// Configure args for mini command
 		// Note: When running via 'go test', os.Args[0] is the test binary
 		// We need to make it look like we're running 'weed mini'
-		os.Args = append([]string{
+		os.Args = []string{
 			"weed",
 			"-dir=" + testDir,
 			"-master.port=" + strconv.Itoa(masterPort),
@@ -206,7 +178,7 @@ func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
 			"-ip=127.0.0.1",
 			"-master.peers=none",     // Faster startup
 			"-s3.iam.readOnly=false", // Enable IAM write operations for tests
-		}, extraArgs...)
+		}
 
 		// Suppress most logging during tests
 		glog.MaxSize = 1024 * 1024
@@ -364,282 +336,6 @@ func testPutObject(t *testing.T, cluster *TestCluster) {
 	t.Logf("✓ Put object: %s/%s (%d bytes)", bucketName, objectKey, len(objectData))
 }
 
-func createTestBucket(t *testing.T, cluster *TestCluster, prefix string) string {
-	bucketName := prefix + randomString(8)
-	_, err := cluster.s3Client.CreateBucket(&s3.CreateBucketInput{
-		Bucket: aws.String(bucketName),
-	})
-	require.NoError(t, err)
-	time.Sleep(100 * time.Millisecond)
-	return bucketName
-}
-
-// generateSSECKey returns a 32-byte key as a raw string (what the SDK expects
-// for SSECustomerKey) and its base64-encoded MD5 (for SSECustomerKeyMD5).
-func generateSSECKey() (keyRaw, keyMD5B64 string) {
-	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
-	key := make([]byte, 32)
-	for i := range key {
-		key[i] = byte(rng.Intn(256))
-	}
-	keyRaw = string(key)
-	keyHash := md5.Sum(key)
-	keyMD5B64 = base64.StdEncoding.EncodeToString(keyHash[:])
-	return
-}
-
-func testPutObjectWithChecksum(t *testing.T, cluster *TestCluster) {
-	bucketName := createTestBucket(t, cluster, "test-put-checksum-")
-	objectKey := "test-checksummed-object.txt"
-	objectData := "Hello, SeaweedFS S3!"
-
-	correctMD5 := calculateMd5(objectData)
-	incorrectMD5 := calculateMd5(objectData + "incorrect")
-
-	// Put object with incorrect MD5 should be rejected
-	_, err := cluster.s3Client.PutObject(&s3.PutObjectInput{
-		Bucket:     aws.String(bucketName),
-		Key:        aws.String(objectKey),
-		Body:       bytes.NewReader([]byte(objectData)),
-		ContentMD5: aws.String(incorrectMD5),
-	})
-	assertBadDigestError(t, err, "PutObject should fail with incorrect MD5")
-
-	t.Logf("✓ Put object with incorrect MD5 rejected: %s/%s", bucketName, objectKey)
-
-	// Put object with correct MD5 should succeed
-	_, err = cluster.s3Client.PutObject(&s3.PutObjectInput{
-		Bucket:     aws.String(bucketName),
-		Key:        aws.String(objectKey),
-		Body:       bytes.NewReader([]byte(objectData)),
-		ContentMD5: aws.String(correctMD5),
-	})
-	require.NoError(t, err, "Failed to put object")
-
-	// Verify object exists
-	headResp, err := cluster.s3Client.HeadObject(&s3.HeadObjectInput{
-		Bucket: aws.String(bucketName),
-		Key:    aws.String(objectKey),
-	})
-	require.NoError(t, err)
-	assert.NotNil(t, headResp.ContentLength)
-	assert.Equal(t, int64(len(objectData)), aws.Int64Value(headResp.ContentLength))
-
-	t.Logf("✓ Put object with correct MD5: %s/%s (%d bytes)", bucketName, objectKey, len(objectData))
-}
-
-// putObjectSSEC sends a PutObject request with SSE-C headers over HTTP.
-// The AWS SDK v1 refuses to send SSE-C keys over plain HTTP, so we use the
-// low-level Request API and clear the Validate handlers to bypass that check.
-// We use Clear() because the specific validator is internal and not easily removable by name.
-func putObjectSSEC(client *s3.S3, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
-	req, output := client.PutObjectRequest(input)
-	req.Handlers.Validate.Clear()
-	err := req.Send()
-	return output, err
-}
-
-func headObjectSSEC(client *s3.S3, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
-	req, output := client.HeadObjectRequest(input)
-	req.Handlers.Validate.Clear()
-	err := req.Send()
-	return output, err
-}
-
-func testPutObjectWithChecksumAndSSEC(t *testing.T, cluster *TestCluster) {
-	bucketName := createTestBucket(t, cluster, "test-put-checksum-ssec-")
-	objectKey := "test-checksummed-ssec-object.txt"
-	objectData := "Hello, SeaweedFS S3 with SSE-C!"
-
-	correctMD5 := calculateMd5(objectData)
-	incorrectMD5 := calculateMd5(objectData + "incorrect")
-	keyRaw, keyMD5B64 := generateSSECKey()
-
-	// Put object with SSE-C and incorrect MD5 should be rejected
-	_, err := putObjectSSEC(cluster.s3Client, &s3.PutObjectInput{
-		Bucket:               aws.String(bucketName),
-		Key:                  aws.String(objectKey),
-		Body:                 bytes.NewReader([]byte(objectData)),
-		ContentMD5:           aws.String(incorrectMD5),
-		SSECustomerAlgorithm: aws.String("AES256"),
-		SSECustomerKey:       aws.String(keyRaw),
-		SSECustomerKeyMD5:    aws.String(keyMD5B64),
-	})
-	assertBadDigestError(t, err, "PutObject with SSE-C should fail with incorrect MD5")
-
-	t.Logf("Put object with SSE-C and incorrect MD5 rejected: %s/%s", bucketName, objectKey)
-
-	// Put object with SSE-C and correct MD5 should succeed
-	_, err = putObjectSSEC(cluster.s3Client, &s3.PutObjectInput{
-		Bucket:               aws.String(bucketName),
-		Key:                  aws.String(objectKey),
-		Body:                 bytes.NewReader([]byte(objectData)),
-		ContentMD5:           aws.String(correctMD5),
-		SSECustomerAlgorithm: aws.String("AES256"),
-		SSECustomerKey:       aws.String(keyRaw),
-		SSECustomerKeyMD5:    aws.String(keyMD5B64),
-	})
-	require.NoError(t, err, "Failed to put object with SSE-C and correct MD5")
-
-	// Verify object exists (SSE-C requires the key for HeadObject too)
-	headResp, err := headObjectSSEC(cluster.s3Client, &s3.HeadObjectInput{
-		Bucket:               aws.String(bucketName),
-		Key:                  aws.String(objectKey),
-		SSECustomerAlgorithm: aws.String("AES256"),
-		SSECustomerKey:       aws.String(keyRaw),
-		SSECustomerKeyMD5:    aws.String(keyMD5B64),
-	})
-	require.NoError(t, err)
-	assert.NotNil(t, headResp.ContentLength)
-	assert.Equal(t, int64(len(objectData)), aws.Int64Value(headResp.ContentLength))
-
-	t.Logf("Put object with SSE-C and correct MD5: %s/%s (%d bytes)", bucketName, objectKey, len(objectData))
-}
-
-func testUploadPartWithChecksum(t *testing.T, cluster *TestCluster) {
-	bucketName := createTestBucket(t, cluster, "test-upload-part-checksum-")
-	objectKey := "test-multipart-checksum.txt"
-	objectData := "Hello, SeaweedFS S3 Multipart!"
-
-	// Initiate multipart upload
-	initResp, err := cluster.s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
-		Bucket: aws.String(bucketName),
-		Key:    aws.String(objectKey),
-	})
-	require.NoError(t, err)
-	uploadID := initResp.UploadId
-
-	correctMD5 := calculateMd5(objectData)
-	incorrectMD5 := calculateMd5(objectData + "incorrect")
-
-	// Upload part with incorrect MD5
-	_, err = cluster.s3Client.UploadPart(&s3.UploadPartInput{
-		Bucket:     aws.String(bucketName),
-		Key:        aws.String(objectKey),
-		PartNumber: aws.Int64(1),
-		UploadId:   uploadID,
-		Body:       bytes.NewReader([]byte(objectData)),
-		ContentMD5: aws.String(incorrectMD5),
-	})
-	assertBadDigestError(t, err, "UploadPart should fail with incorrect MD5")
-
-	// Upload part with correct MD5
-	partResp, err := cluster.s3Client.UploadPart(&s3.UploadPartInput{
-		Bucket:     aws.String(bucketName),
-		Key:        aws.String(objectKey),
-		PartNumber: aws.Int64(1),
-		UploadId:   uploadID,
-		Body:       bytes.NewReader([]byte(objectData)),
-		ContentMD5: aws.String(correctMD5),
-	})
-	require.NoError(t, err, "Failed to upload part with correct MD5")
-
-	// Complete multipart upload
-	_, err = cluster.s3Client.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
-		Bucket:   aws.String(bucketName),
-		Key:      aws.String(objectKey),
-		UploadId: uploadID,
-		MultipartUpload: &s3.CompletedMultipartUpload{
-			Parts: []*s3.CompletedPart{
-				{
-					ETag:       partResp.ETag,
-					PartNumber: aws.Int64(1),
-				},
-			},
-		},
-	})
-	require.NoError(t, err, "Failed to complete multipart upload")
-
-	// Verify object exists
-	headResp, err := cluster.s3Client.HeadObject(&s3.HeadObjectInput{
-		Bucket: aws.String(bucketName),
-		Key:    aws.String(objectKey),
-	})
-	require.NoError(t, err)
-	assert.Equal(t, int64(len(objectData)), aws.Int64Value(headResp.ContentLength))
-
-	t.Logf("✓ Multipart upload with checksum successful: %s/%s", bucketName, objectKey)
-}
-
-func testPutPartWithChecksum(t *testing.T, cluster *TestCluster) {
-	bucketName := createTestBucket(t, cluster, "test-put-checksum-")
-	objectKey := "test-checksummed-part.txt"
-
-	partData := "Hello, SeaweedFS S3!"
-
-	correctMD5 := calculateMd5(partData)
-	incorrectMD5 := calculateMd5(partData + "incorrect")
-
-	createResp, err := cluster.s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
-		Bucket: aws.String(bucketName),
-		Key:    aws.String(objectKey),
-	})
-	require.NoError(t, err)
-
-	uploadID := createResp.UploadId
-
-	partBody := []byte(partData)
-
-	_, err = cluster.s3Client.UploadPart(&s3.UploadPartInput{
-		Bucket:     aws.String(bucketName),
-		Key:        aws.String(objectKey),
-		UploadId:   uploadID,
-		PartNumber: aws.Int64(1),
-		Body:       bytes.NewReader(partBody),
-		ContentMD5: aws.String(incorrectMD5),
-	})
-	assertBadDigestError(t, err, "UploadPart should fail with incorrect MD5")
-
-	uploadResp, err := cluster.s3Client.UploadPart(&s3.UploadPartInput{
-		Bucket:     aws.String(bucketName),
-		Key:        aws.String(objectKey),
-		UploadId:   uploadID,
-		PartNumber: aws.Int64(1),
-		Body:       bytes.NewReader(partBody),
-		ContentMD5: aws.String(correctMD5),
-	})
-	require.NoError(t, err)
-
-	_, err = cluster.s3Client.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
-		Bucket:   aws.String(bucketName),
-		Key:      aws.String(objectKey),
-		UploadId: uploadID,
-		MultipartUpload: &s3.CompletedMultipartUpload{
-			Parts: []*s3.CompletedPart{
-				{
-					ETag:       uploadResp.ETag,
-					PartNumber: aws.Int64(1),
-				},
-			},
-		},
-	})
-	require.NoError(t, err, "Failed to complete multipart upload")
-
-	// Verify object exists
-	headResp, err := cluster.s3Client.HeadObject(&s3.HeadObjectInput{
-		Bucket: aws.String(bucketName),
-		Key:    aws.String(objectKey),
-	})
-	require.NoError(t, err)
-	assert.Equal(t, int64(len(partData)), aws.Int64Value(headResp.ContentLength))
-
-	t.Logf("✓ UploadPart with MD5 validation: %s/%s", bucketName, objectKey)
-}
-
-func calculateMd5(objectData string) string {
-	dataBytes := []byte(objectData)
-	hash := md5.Sum(dataBytes)
-	return base64.StdEncoding.EncodeToString(hash[:])
-}
-
-func assertBadDigestError(t *testing.T, err error, description string) {
-	require.Error(t, err, description)
-
-	var awsErr awserr.Error
-	require.ErrorAs(t, err, &awsErr)
-	assert.Equal(t, "BadDigest", awsErr.Code())
-}
-
 func testGetObject(t *testing.T, cluster *TestCluster) {
 	bucketName := "test-get-" + randomString(8)
 	objectKey := "test-data.txt"
@@ -780,49 +476,6 @@ func testDeleteBucket(t *testing.T, cluster *TestCluster) {
 	t.Logf("✓ Deleted bucket: %s", bucketName)
 }
 
-func TestS3DeleteBucketNotEmpty(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	cluster, err := startMiniCluster(t, "-s3.allowDeleteBucketNotEmpty=false")
-	require.NoError(t, err)
-	defer cluster.Stop()
-
-	t.Run("DeleteNonEmptyBucketFails", func(t *testing.T) {
-		bucketName := createTestBucket(t, cluster, "test-notempty-")
-		objectKey := "keep-me.txt"
-
-		// Put an object so the bucket is non-empty
-		_, err := cluster.s3Client.PutObject(&s3.PutObjectInput{
-			Bucket: aws.String(bucketName),
-			Key:    aws.String(objectKey),
-			Body:   bytes.NewReader([]byte("data")),
-		})
-		require.NoError(t, err)
-
-		// Attempt to delete the non-empty bucket — must fail with BucketNotEmpty (409)
-		_, err = cluster.s3Client.DeleteBucket(&s3.DeleteBucketInput{
-			Bucket: aws.String(bucketName),
-		})
-		require.Error(t, err, "deleting a non-empty bucket should fail")
-		var awsErr awserr.Error
-		require.ErrorAs(t, err, &awsErr)
-		assert.Equal(t, "BucketNotEmpty", awsErr.Code(),
-			"expected BucketNotEmpty error code, got %s: %s", awsErr.Code(), awsErr.Message())
-	})
-
-	t.Run("DeleteEmptyBucketSucceeds", func(t *testing.T) {
-		bucketName := createTestBucket(t, cluster, "test-empty-")
-
-		// Delete the empty bucket — should succeed even with the flag
-		_, err := cluster.s3Client.DeleteBucket(&s3.DeleteBucketInput{
-			Bucket: aws.String(bucketName),
-		})
-		require.NoError(t, err, "deleting an empty bucket should succeed")
-	})
-}
-
 // randomString generates a random string for unique naming
 func randomString(length int) string {
 	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"

test/s3/policy/policy_test.go

@@ -2,7 +2,6 @@ package policy
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -15,11 +14,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/seaweedfs/seaweedfs/weed/command"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -136,151 +130,6 @@ func TestS3PolicyShellRevised(t *testing.T) {
 	}
 }
 
-func TestS3IAMAttachDetachUserPolicy(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	cluster, err := startMiniCluster(t)
-	require.NoError(t, err)
-	defer cluster.Stop()
-
-	time.Sleep(500 * time.Millisecond)
-
-	policyName := uniqueName("managed-policy")
-	policyArn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
-	policyContent := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]}`
-	tmpPolicyFile, err := os.CreateTemp("", "test_policy_attach_*.json")
-	require.NoError(t, err)
-	defer os.Remove(tmpPolicyFile.Name())
-	_, err = tmpPolicyFile.WriteString(policyContent)
-	require.NoError(t, err)
-	require.NoError(t, tmpPolicyFile.Close())
-
-	weedCmd := "weed"
-	masterAddr := string(pb.NewServerAddress("127.0.0.1", cluster.masterPort, cluster.masterGrpcPort))
-	filerAddr := string(pb.NewServerAddress("127.0.0.1", cluster.filerPort, cluster.filerGrpcPort))
-	execShell(t, weedCmd, masterAddr, filerAddr, fmt.Sprintf("s3.policy -put -name=%s -file=%s", policyName, tmpPolicyFile.Name()))
-
-	iamClient := newIAMClient(t, cluster.s3Endpoint)
-
-	userName := uniqueName("iam-user")
-	_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
-	require.NoError(t, err)
-
-	_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
-		UserName:  aws.String(userName),
-		PolicyArn: aws.String(policyArn),
-	})
-	require.NoError(t, err)
-
-	listOut, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
-		UserName: aws.String(userName),
-	})
-	require.NoError(t, err)
-	require.True(t, attachedPolicyContains(listOut.AttachedPolicies, policyName))
-
-	_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
-		UserName:  aws.String(userName),
-		PolicyArn: aws.String(policyArn),
-	})
-	require.NoError(t, err)
-
-	listOut, err = iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
-		UserName: aws.String(userName),
-	})
-	require.NoError(t, err)
-	require.False(t, attachedPolicyContains(listOut.AttachedPolicies, policyName))
-
-	_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
-		UserName:  aws.String(userName),
-		PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
-	})
-	require.Error(t, err)
-	if awsErr, ok := err.(awserr.Error); ok {
-		require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
-	}
-}
-
-func TestS3IAMListPoliciesAndGetPolicy(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	cluster, err := startMiniCluster(t)
-	require.NoError(t, err)
-	defer cluster.Stop()
-
-	time.Sleep(500 * time.Millisecond)
-
-	policyName := uniqueName("managed-policy")
-	policyArn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
-	policyContent := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListAllMyBuckets","Resource":"*"}]}`
-
-	iamClient := newIAMClient(t, cluster.s3Endpoint)
-	_, err = iamClient.CreatePolicy(&iam.CreatePolicyInput{
-		PolicyName:     aws.String(policyName),
-		PolicyDocument: aws.String(policyContent),
-	})
-	require.NoError(t, err)
-
-	listOut, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
-	require.NoError(t, err)
-	require.True(t, managedPolicyContains(listOut.Policies, policyName))
-
-	getOut, err := iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
-	require.NoError(t, err)
-	require.NotNil(t, getOut.Policy)
-	require.NotNil(t, getOut.Policy.PolicyName)
-	require.Equal(t, policyName, *getOut.Policy.PolicyName)
-
-	missingArn := fmt.Sprintf("arn:aws:iam:::policy/%s", uniqueName("missing"))
-	_, err = iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(missingArn)})
-	require.Error(t, err)
-	var awsErr awserr.Error
-	require.True(t, errors.As(err, &awsErr))
-	require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
-}
-
-func TestS3IAMDeletePolicyInUse(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	cluster, err := startMiniCluster(t)
-	require.NoError(t, err)
-	defer cluster.Stop()
-
-	time.Sleep(500 * time.Millisecond)
-
-	policyName := uniqueName("managed-delete-policy")
-	policyArn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
-	policyContent := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}`
-
-	iamClient := newIAMClient(t, cluster.s3Endpoint)
-	_, err = iamClient.CreatePolicy(&iam.CreatePolicyInput{
-		PolicyName:     aws.String(policyName),
-		PolicyDocument: aws.String(policyContent),
-	})
-	require.NoError(t, err)
-
-	userName := uniqueName("iam-user-delete-policy")
-	_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
-	require.NoError(t, err)
-
-	_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
-		UserName:  aws.String(userName),
-		PolicyArn: aws.String(policyArn),
-	})
-	require.NoError(t, err)
-
-	_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
-	require.Error(t, err)
-	var awsErr awserr.Error
-	require.True(t, errors.As(err, &awsErr))
-	require.Equal(t, iam.ErrCodeDeleteConflictException, awsErr.Code())
-}
-
 func execShell(t *testing.T, weedCmd, master, filer, shellCmd string) string {
 	// weed shell -master=... -filer=...
 	args := []string{"shell", "-master=" + master, "-filer=" + filer}
@@ -296,52 +145,6 @@ func execShell(t *testing.T, weedCmd, master, filer, shellCmd string) string {
 	return string(out)
 }
 
-func newIAMClient(t *testing.T, endpoint string) *iam.IAM {
-	t.Helper()
-
-	accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
-	secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
-	if accessKey == "" {
-		accessKey = "admin"
-	}
-	if secretKey == "" {
-		secretKey = "admin"
-	}
-
-	sess, err := session.NewSession(&aws.Config{
-		Region:           aws.String("us-east-1"),
-		Endpoint:         aws.String(endpoint),
-		DisableSSL:       aws.Bool(true),
-		S3ForcePathStyle: aws.Bool(true),
-		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
-	})
-	require.NoError(t, err)
-
-	return iam.New(sess)
-}
-
-func attachedPolicyContains(policies []*iam.AttachedPolicy, policyName string) bool {
-	for _, policy := range policies {
-		if policy.PolicyName != nil && *policy.PolicyName == policyName {
-			return true
-		}
-	}
-	return false
-}
-
-func managedPolicyContains(policies []*iam.Policy, policyName string) bool {
-	for _, policy := range policies {
-		if policy.PolicyName != nil && *policy.PolicyName == policyName {
-			return true
-		}
-	}
-	return false
-}
-
-func uniqueName(prefix string) string {
-	return fmt.Sprintf("%s-%s", prefix, strconv.FormatInt(time.Now().UnixNano(), 36))
-}
-
 // --- Test setup helpers ---
 
 func findAvailablePort() (int, error) {
@@ -413,14 +216,6 @@ enabled = true
 	err = os.WriteFile(credentialToml, []byte(credentialConfig), 0644)
 	require.NoError(t, err)
 
-	// Set environment variables for admin credentials safely for this test
-	if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
-		t.Setenv("AWS_ACCESS_KEY_ID", "admin")
-	}
-	if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
-		t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
-	}
-
 	cluster.wg.Add(1)
 	go func() {
 		defer cluster.wg.Done()
@@ -447,7 +242,6 @@ enabled = true
 			"-master.volumeSizeLimitMB=32",
 			"-ip=127.0.0.1",
 			"-master.peers=none",
-			"-s3.iam.readOnly=false",
 		}
 		glog.MaxSize = 1024 * 1024
 		for _, cmd := range command.Commands {

test/s3/sse/s3_volume_encryption_test.go

@@ -1,4 +1,5 @@
 //go:build integration
+// +build integration
 
 package sse
 

test/s3tables/catalog/iceberg_catalog_test.go

@@ -6,7 +6,6 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"flag"
 	"fmt"
 	"io"
 	"net"
@@ -19,37 +18,6 @@ import (
 	"time"
 )
 
-// sharedEnv is the single TestEnvironment shared across all tests in this package.
-var sharedEnv *TestEnvironment
-
-// TestMain starts one weed mini instance for the whole package and tears it down
-// after all tests have run.
-func TestMain(m *testing.M) {
-	flag.Parse()
-	if os.Getenv("SHORT") != "" || testing.Short() {
-		// Let tests self-skip when run with -short.
-		os.Exit(m.Run())
-	}
-
-	env, err := newTestEnvironmentForMain()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "SKIP: setup failed: %v\n", err)
-		os.Exit(0) // Skip all tests rather than fail
-	}
-	sharedEnv = env
-
-	if startErr := sharedEnv.startSeaweedFSForMain(); startErr != nil {
-		fmt.Fprintf(os.Stderr, "SKIP: weed mini failed to start: %v\n", startErr)
-		sharedEnv.cleanupForMain()
-		os.Exit(0)
-	}
-
-	code := m.Run()
-
-	sharedEnv.cleanupForMain()
-	os.Exit(code)
-}
-
 // TestEnvironment contains the test environment configuration
 type TestEnvironment struct {
 	seaweedDir      string
@@ -86,15 +54,17 @@ func getFreePort() (int, net.Listener, error) {
 	return addr.Port, listener, nil
 }
 
-// newTestEnvironmentForMain creates a TestEnvironment without calling t.Fatalf so it
-// can be used from TestMain (which has no *testing.T).
-func newTestEnvironmentForMain() (*TestEnvironment, error) {
+// NewTestEnvironment creates a new test environment
+func NewTestEnvironment(t *testing.T) *TestEnvironment {
+	t.Helper()
+
 	// Find the SeaweedFS root directory
 	wd, err := os.Getwd()
 	if err != nil {
-		return nil, fmt.Errorf("get working directory: %w", err)
+		t.Fatalf("Failed to get working directory: %v", err)
 	}
 
+	// Navigate up to find the SeaweedFS root (contains go.mod)
 	seaweedDir := wd
 	for i := 0; i < 5; i++ {
 		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
@@ -106,93 +76,82 @@ func newTestEnvironmentForMain() (*TestEnvironment, error) {
 	// Check for weed binary
 	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
 	if _, err := os.Stat(weedBinary); os.IsNotExist(err) {
+		// Try system PATH
 		weedBinary = "weed"
 		if _, err := exec.LookPath(weedBinary); err != nil {
-			return nil, fmt.Errorf("weed binary not found")
+			t.Skip("weed binary not found, skipping integration test")
 		}
 	}
 
 	// Create temporary data directory
 	dataDir, err := os.MkdirTemp("", "seaweed-iceberg-test-*")
 	if err != nil {
-		return nil, fmt.Errorf("create temp dir: %w", err)
+		t.Fatalf("Failed to create temp dir: %v", err)
 	}
 
 	// Allocate free ephemeral ports for each service
 	var listeners []net.Listener
-	closeListeners := func() {
+	defer func() {
 		for _, l := range listeners {
 			l.Close()
 		}
-	}
+	}()
 
 	var l net.Listener
 	s3Port, l, err := getFreePort()
 	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for S3: %w", err)
+		t.Fatalf("Failed to get free port for S3: %v", err)
 	}
 	listeners = append(listeners, l)
 
 	icebergPort, l, err := getFreePort()
 	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Iceberg: %w", err)
+		t.Fatalf("Failed to get free port for Iceberg: %v", err)
 	}
 	listeners = append(listeners, l)
 
 	s3GrpcPort, l, err := getFreePort()
 	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for S3 gRPC: %w", err)
+		t.Fatalf("Failed to get free port for S3 gRPC: %v", err)
 	}
 	listeners = append(listeners, l)
 
 	masterPort, l, err := getFreePort()
 	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Master: %w", err)
+		t.Fatalf("Failed to get free port for Master: %v", err)
 	}
 	listeners = append(listeners, l)
 
 	masterGrpcPort, l, err := getFreePort()
 	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Master gRPC: %w", err)
+		t.Fatalf("Failed to get free port for Master gRPC: %v", err)
 	}
 	listeners = append(listeners, l)
 
 	filerPort, l, err := getFreePort()
 	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Filer: %w", err)
+		t.Fatalf("Failed to get free port for Filer: %v", err)
 	}
 	listeners = append(listeners, l)
 
 	filerGrpcPort, l, err := getFreePort()
 	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Filer gRPC: %w", err)
+		t.Fatalf("Failed to get free port for Filer gRPC: %v", err)
 	}
 	listeners = append(listeners, l)
 
 	volumePort, l, err := getFreePort()
 	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Volume: %w", err)
+		t.Fatalf("Failed to get free port for Volume: %v", err)
 	}
 	listeners = append(listeners, l)
 
 	volumeGrpcPort, l, err := getFreePort()
 	if err != nil {
-		closeListeners()
-		return nil, fmt.Errorf("get free port for Volume gRPC: %w", err)
+		t.Fatalf("Failed to get free port for Volume gRPC: %v", err)
 	}
 	listeners = append(listeners, l)
 
-	// Release the port reservations so weed mini can bind to them
-	closeListeners()
-
 	return &TestEnvironment{
 		seaweedDir:      seaweedDir,
 		weedBinary:      weedBinary,
@@ -207,11 +166,13 @@ func newTestEnvironmentForMain() (*TestEnvironment, error) {
 		volumePort:      volumePort,
 		volumeGrpcPort:  volumeGrpcPort,
 		dockerAvailable: hasDocker(),
-	}, nil
+	}
 }
 
-// startSeaweedFSForMain starts weed mini without a *testing.T (for use in TestMain).
-func (env *TestEnvironment) startSeaweedFSForMain() error {
+// StartSeaweedFS starts a SeaweedFS mini cluster
+func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
+	t.Helper()
+
 	ctx, cancel := context.WithCancel(context.Background())
 	env.weedCancel = cancel
 
@@ -221,8 +182,7 @@ func (env *TestEnvironment) startSeaweedFSForMain() error {
 
 	for _, dir := range []string{masterDir, filerDir, volumeDir} {
 		if err := os.MkdirAll(dir, 0755); err != nil {
-			cancel()
-			return fmt.Errorf("create directory %s: %w", dir, err)
+			t.Fatalf("Failed to create directory %s: %v", dir, err)
 		}
 	}
 
@@ -243,30 +203,13 @@ func (env *TestEnvironment) startSeaweedFSForMain() error {
 	cmd.Stderr = os.Stderr
 
 	if err := cmd.Start(); err != nil {
-		cancel()
-		return fmt.Errorf("start SeaweedFS: %w", err)
+		t.Fatalf("Failed to start SeaweedFS: %v", err)
 	}
 	env.weedProcess = cmd
 
+	// Wait for services to be ready
 	if !env.waitForService(fmt.Sprintf("http://127.0.0.1:%d/v1/config", env.icebergPort), 30*time.Second) {
-		cancel()
-		cmd.Wait()
-		return fmt.Errorf("Iceberg REST API did not become ready")
-	}
-	return nil
-}
-
-// cleanupForMain stops SeaweedFS and cleans up resources (no *testing.T needed).
-func (env *TestEnvironment) cleanupForMain() {
-	if env.weedCancel != nil {
-		env.weedCancel()
-	}
-	if env.weedProcess != nil {
-		time.Sleep(2 * time.Second)
-		env.weedProcess.Wait()
-	}
-	if env.dataDir != "" {
-		os.RemoveAll(env.dataDir)
+		t.Fatalf("Iceberg REST API did not become ready")
 	}
 }
 
@@ -287,6 +230,25 @@ func (env *TestEnvironment) waitForService(url string, timeout time.Duration) bo
 	return false
 }
 
+// Cleanup stops SeaweedFS and cleans up resources
+func (env *TestEnvironment) Cleanup(t *testing.T) {
+	t.Helper()
+
+	if env.weedCancel != nil {
+		env.weedCancel()
+	}
+
+	if env.weedProcess != nil {
+		// Give process time to shut down gracefully
+		time.Sleep(2 * time.Second)
+		env.weedProcess.Wait()
+	}
+
+	if env.dataDir != "" {
+		os.RemoveAll(env.dataDir)
+	}
+}
+
 // IcebergURL returns the Iceberg REST Catalog URL
 func (env *TestEnvironment) IcebergURL() string {
 	return fmt.Sprintf("http://127.0.0.1:%d", env.icebergPort)
@@ -298,7 +260,10 @@ func TestIcebergConfig(t *testing.T) {
 		t.Skip("Skipping integration test in short mode")
 	}
 
-	env := sharedEnv
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	env.StartSeaweedFS(t)
 
 	// Test GET /v1/config
 	resp, err := http.Get(env.IcebergURL() + "/v1/config")
@@ -329,14 +294,16 @@ func TestIcebergNamespaces(t *testing.T) {
 		t.Skip("Skipping integration test in short mode")
 	}
 
-	env := sharedEnv
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	env.StartSeaweedFS(t)
 
 	// Create the default table bucket first via S3
-	bucketName := "warehouse-ns-" + randomSuffix()
-	createTableBucket(t, env, bucketName)
+	createTableBucket(t, env, "warehouse")
 
 	// Test GET /v1/namespaces (should return empty list initially)
-	resp, err := http.Get(env.IcebergURL() + icebergPath(bucketName, "/v1/namespaces"))
+	resp, err := http.Get(env.IcebergURL() + "/v1/namespaces")
 	if err != nil {
 		t.Fatalf("Failed to list namespaces: %v", err)
 	}
@@ -354,14 +321,16 @@ func TestStageCreateAndFinalizeFlow(t *testing.T) {
 		t.Skip("Skipping integration test in short mode")
 	}
 
-	env := sharedEnv
-	bucketName := "warehouse-stage-" + randomSuffix()
-	createTableBucket(t, env, bucketName)
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
 
-	namespace := "stage_ns_" + randomSuffix()
+	env.StartSeaweedFS(t)
+	createTableBucket(t, env, "warehouse")
+
+	namespace := "stage_ns"
 	tableName := "orders"
 
-	status, _, err := doIcebergJSONRequest(env, http.MethodPost, icebergPath(bucketName, "/v1/namespaces"), map[string]any{
+	status, _, err := doIcebergJSONRequest(env, http.MethodPost, "/v1/namespaces", map[string]any{
 		"namespace": []string{namespace},
 	})
 	if err != nil {
@@ -371,7 +340,7 @@ func TestStageCreateAndFinalizeFlow(t *testing.T) {
 		t.Fatalf("Create namespace status = %d, want 200 or 409", status)
 	}
 
-	status, badReqResp, err := doIcebergJSONRequest(env, http.MethodPost, icebergPath(bucketName, fmt.Sprintf("/v1/namespaces/%s/tables", namespace)), map[string]any{
+	status, badReqResp, err := doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables", namespace), map[string]any{
 		"stage-create": true,
 	})
 	if err != nil {
@@ -389,7 +358,7 @@ func TestStageCreateAndFinalizeFlow(t *testing.T) {
 		t.Fatalf("error.message = %v, want it to include %q", errorObj["message"], "table name is required")
 	}
 
-	status, stageResp, err := doIcebergJSONRequest(env, http.MethodPost, icebergPath(bucketName, fmt.Sprintf("/v1/namespaces/%s/tables", namespace)), map[string]any{
+	status, stageResp, err := doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables", namespace), map[string]any{
 		"name":         tableName,
 		"stage-create": true,
 	})
@@ -404,7 +373,7 @@ func TestStageCreateAndFinalizeFlow(t *testing.T) {
 		t.Fatalf("stage metadata-location = %q, want suffix /metadata/v1.metadata.json", stageLocation)
 	}
 
-	status, _, err = doIcebergJSONRequest(env, http.MethodGet, icebergPath(bucketName, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName)), nil)
+	status, _, err = doIcebergJSONRequest(env, http.MethodGet, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), nil)
 	if err != nil {
 		t.Fatalf("Load staged table request failed: %v", err)
 	}
@@ -412,7 +381,7 @@ func TestStageCreateAndFinalizeFlow(t *testing.T) {
 		t.Fatalf("Load staged table status = %d, want 404", status)
 	}
 
-	status, commitResp, err := doIcebergJSONRequest(env, http.MethodPost, icebergPath(bucketName, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName)), map[string]any{
+	status, commitResp, err := doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), map[string]any{
 		"requirements": []map[string]any{
 			{"type": "assert-create"},
 		},
@@ -429,7 +398,7 @@ func TestStageCreateAndFinalizeFlow(t *testing.T) {
 		t.Fatalf("final metadata-location = %q, want suffix /metadata/v1.metadata.json", commitLocation)
 	}
 
-	status, loadResp, err := doIcebergJSONRequest(env, http.MethodGet, icebergPath(bucketName, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName)), nil)
+	status, loadResp, err := doIcebergJSONRequest(env, http.MethodGet, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), nil)
 	if err != nil {
 		t.Fatalf("Load finalized table request failed: %v", err)
 	}
@@ -448,14 +417,16 @@ func TestCommitMissingTableWithoutAssertCreate(t *testing.T) {
 		t.Skip("Skipping integration test in short mode")
 	}
 
-	env := sharedEnv
-	bucketName := "warehouse-missing-" + randomSuffix()
-	createTableBucket(t, env, bucketName)
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
 
-	namespace := "stage_missing_assert_ns_" + randomSuffix()
+	env.StartSeaweedFS(t)
+	createTableBucket(t, env, "warehouse")
+
+	namespace := "stage_missing_assert_ns"
 	tableName := "missing_table"
 
-	status, _, err := doIcebergJSONRequest(env, http.MethodPost, icebergPath(bucketName, "/v1/namespaces"), map[string]any{
+	status, _, err := doIcebergJSONRequest(env, http.MethodPost, "/v1/namespaces", map[string]any{
 		"namespace": []string{namespace},
 	})
 	if err != nil {
@@ -465,7 +436,7 @@ func TestCommitMissingTableWithoutAssertCreate(t *testing.T) {
 		t.Fatalf("Create namespace status = %d, want 200 or 409", status)
 	}
 
-	status, _, err = doIcebergJSONRequest(env, http.MethodPost, icebergPath(bucketName, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName)), map[string]any{
+	status, _, err = doIcebergJSONRequest(env, http.MethodPost, fmt.Sprintf("/v1/namespaces/%s/tables/%s", namespace, tableName), map[string]any{
 		"requirements": []any{},
 		"updates":      []any{},
 	})
@@ -520,21 +491,6 @@ func doIcebergJSONRequest(env *TestEnvironment, method, path string, payload any
 	return resp.StatusCode, decoded, nil
 }
 
-// icebergPath inserts the table bucket prefix into Iceberg REST API paths.
-// For example, "/v1/namespaces" with prefix "my-bucket" becomes
-// "/v1/my-bucket/namespaces".
-func icebergPath(prefix, path string) string {
-	if prefix == "" {
-		return path
-	}
-	const base = "/v1/"
-	if !strings.HasPrefix(path, base) {
-		return path
-	}
-	withPrefix := base + prefix + "/" + strings.TrimPrefix(path, base)
-	return withPrefix
-}
-
 // createTableBucket creates a table bucket via the S3Tables REST API
 func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
 	t.Helper()
@@ -555,20 +511,13 @@ func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
 	}
 	defer resp.Body.Close()
 
-	body, _ := io.ReadAll(resp.Body)
-	t.Logf("Create table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
-
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusConflict {
+		body, _ := io.ReadAll(resp.Body)
 		t.Fatalf("Failed to create table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
 	}
 	t.Logf("Created table bucket %s", bucketName)
 }
 
-// randomSuffix returns a short random hex suffix for unique resource naming.
-func randomSuffix() string {
-	return fmt.Sprintf("%x", time.Now().UnixNano()&0xffffffff)
-}
-
 // TestDuckDBIntegration tests Iceberg catalog operations using DuckDB
 // This test requires Docker to be available
 func TestDuckDBIntegration(t *testing.T) {
@@ -576,12 +525,15 @@ func TestDuckDBIntegration(t *testing.T) {
 		t.Skip("Skipping integration test in short mode")
 	}
 
-	env := sharedEnv
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
 
 	if !env.dockerAvailable {
 		t.Skip("Docker not available, skipping DuckDB integration test")
 	}
 
+	env.StartSeaweedFS(t)
+
 	// Create a temporary SQL file for DuckDB to execute
 	sqlFile := filepath.Join(env.dataDir, "test.sql")
 	sqlContent := fmt.Sprintf(`

test/s3tables/catalog/pyiceberg_test.go

@@ -26,14 +26,17 @@ func TestPyIcebergRestCatalog(t *testing.T) {
 		t.Skip("Skipping integration test in short mode")
 	}
 
-	env := sharedEnv
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
 
 	if !env.dockerAvailable {
 		t.Skip("Docker not available, skipping PyIceberg integration test")
 	}
 
+	env.StartSeaweedFS(t)
+
 	// Create the test bucket first
-	bucketName := "pyiceberg-compat-test-" + randomSuffix()
+	bucketName := "pyiceberg-compat-test"
 	createTableBucket(t, env, bucketName)
 
 	// Build the test working directory path
@@ -53,6 +56,8 @@ func TestPyIcebergRestCatalog(t *testing.T) {
 
 	cmd := exec.Command("docker", "run", "--rm",
 		"--add-host", "host.docker.internal:host-gateway",
+		"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", "test"),
+		"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", "test"),
 		"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
 		"-v", fmt.Sprintf("%s:/app:ro", testDir),
 		"iceberg-rest-test",
@@ -73,65 +78,3 @@ func TestPyIcebergRestCatalog(t *testing.T) {
 		t.Errorf("PyIceberg test failed: %v", err)
 	}
 }
-
-// TestPyIcebergRestCatalogAuthenticated tests the Iceberg REST Catalog using PyIceberg with authentication.
-// This test uses the default admin credentials that SeaweedFS creates on startup.
-func TestPyIcebergRestCatalogAuthenticated(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	env := sharedEnv
-
-	if !env.dockerAvailable {
-		t.Skip("Docker not available, skipping PyIceberg integration test")
-	}
-
-	// Use default admin credentials
-	testAccessKey := "admin"
-	testSecretKey := "admin"
-
-	// Create the test bucket first (using unauthenticated request, which works with DefaultAllow)
-	bucketName := "pyiceberg-auth-test-" + randomSuffix()
-	createTableBucket(t, env, bucketName)
-
-	// Build the test working directory path
-	testDir := filepath.Join(env.seaweedDir, "test", "s3tables", "catalog")
-
-	// Run PyIceberg test using Docker with authentication
-	catalogURL := fmt.Sprintf("http://host.docker.internal:%d", env.icebergPort)
-	s3Endpoint := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
-	warehouse := fmt.Sprintf("s3://%s/", bucketName)
-
-	// Build the test image first for faster repeated runs
-	buildCmd := exec.Command("docker", "build", "-t", "iceberg-rest-test", "-f", "Dockerfile.pyiceberg", ".")
-	buildCmd.Dir = testDir
-	if out, err := buildCmd.CombinedOutput(); err != nil {
-		t.Fatalf("Failed to build test image: %v\n%s", err, string(out))
-	}
-
-	cmd := exec.Command("docker", "run", "--rm",
-		"--add-host", "host.docker.internal:host-gateway",
-		"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
-		"-v", fmt.Sprintf("%s:/app:ro", testDir),
-		"iceberg-rest-test",
-		"python3", "/app/test_rest_catalog_auth.py",
-		"--catalog-url", catalogURL,
-		"--warehouse", warehouse,
-		"--prefix", bucketName,
-		"--access-key", testAccessKey,
-		"--secret-key", testSecretKey,
-	)
-	cmd.Dir = testDir
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-
-	t.Logf("Running PyIceberg REST catalog test with authentication...")
-	t.Logf("  Catalog URL: %s", catalogURL)
-	t.Logf("  Warehouse: %s", warehouse)
-	t.Logf("  Access Key: %s", testAccessKey)
-
-	if err := cmd.Run(); err != nil {
-		t.Errorf("PyIceberg authenticated test failed: %v", err)
-	}
-}

test/s3tables/catalog/pyiceberg_test_helpers_test.go

@@ -1,36 +0,0 @@
-package catalog
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"testing"
-)
-
-// verifyTableBucketMetadata verifies that a table bucket was created with proper metadata
-func verifyTableBucketMetadata(t *testing.T, env *TestEnvironment, bucketName string) {
-	t.Helper()
-
-	// Use S3Tables REST API to get the bucket
-	endpoint := fmt.Sprintf("http://localhost:%d/buckets/%s", env.s3Port, bucketName)
-
-	req, err := http.NewRequest(http.MethodGet, endpoint, nil)
-	if err != nil {
-		t.Fatalf("Failed to create request: %v", err)
-	}
-	req.Header.Set("Content-Type", "application/x-amz-json-1.1")
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		t.Fatalf("Failed to get table bucket %s: %v", bucketName, err)
-	}
-	defer resp.Body.Close()
-
-	body, _ := io.ReadAll(resp.Body)
-	t.Logf("Get table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
-
-	if resp.StatusCode != http.StatusOK {
-		t.Fatalf("Failed to get table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
-	}
-	t.Logf("Verified table bucket %s exists with metadata", bucketName)
-}

test/s3tables/catalog/test_rest_catalog.py

@@ -201,7 +201,6 @@ def main():
                     "uri": args.catalog_url,
                     "warehouse": args.warehouse,
                     "prefix": args.prefix,
-                    "s3.anonymous": "true",  # Disable AWS request signing for unauthenticated access
                 }
             )
             print(f"Successfully connected to catalog on attempt {attempt + 1}")

test/s3tables/catalog/test_rest_catalog_auth.py

@@ -1,223 +0,0 @@
-#!/usr/bin/env python3
-"""
-Iceberg REST Catalog Compatibility Test for SeaweedFS (Authenticated)
-
-This script tests the Iceberg REST Catalog API compatibility with authentication.
-
-Usage:
-    python3 test_rest_catalog_auth.py --catalog-url http://localhost:8182 \\
-        --access-key admin --secret-key admin
-
-Requirements:
-    pip install pyiceberg[s3fs]
-"""
-
-import argparse
-import sys
-from pyiceberg.catalog import load_catalog
-from pyiceberg.schema import Schema
-from pyiceberg.types import (
-    IntegerType,
-    LongType,
-    StringType,
-    NestedField,
-)
-from pyiceberg.exceptions import (
-    NamespaceAlreadyExistsError,
-    NoSuchNamespaceError,
-    TableAlreadyExistsError,
-    NoSuchTableError,
-)
-
-
-def test_config_endpoint(catalog):
-    """Test that the catalog config endpoint returns valid configuration."""
-    print("Testing /v1/config endpoint...")
-    # The catalog is already loaded which means config endpoint worked
-    print("  /v1/config endpoint working")
-    return True
-
-
-def test_namespace_operations(catalog, prefix):
-    """Test namespace CRUD operations."""
-    print("Testing namespace operations...")
-    namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
-    
-    # List initial namespaces
-    namespaces = catalog.list_namespaces()
-    print(f"  Initial namespaces: {namespaces}")
-    
-    # Create namespace
-    try:
-        catalog.create_namespace(namespace)
-        print(f"  Created namespace: {namespace}")
-    except NamespaceAlreadyExistsError:
-        print(f"  ! Namespace already exists: {namespace}")
-    
-    # List namespaces (should include our new one)
-    namespaces = catalog.list_namespaces()
-    if namespace in namespaces:
-        print("  Namespace appears in list")
-    else:
-        print(f"  Namespace not found in list: {namespaces}")
-        return False
-    
-    # Get namespace properties
-    try:
-        props = catalog.load_namespace_properties(namespace)
-        print(f"  Loaded namespace properties: {props}")
-    except NoSuchNamespaceError:
-        print(f"  Failed to load namespace properties")
-        return False
-    
-    return True
-
-
-def test_table_operations(catalog, prefix):
-    """Test table CRUD operations."""
-    print("Testing table operations...")
-    namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
-    table_name = "auth_test_table"
-    table_id = namespace + (table_name,)
-    
-    # Define a simple schema
-    schema = Schema(
-        NestedField(field_id=1, name="id", field_type=LongType(), required=True),
-        NestedField(field_id=2, name="name", field_type=StringType(), required=False),
-        NestedField(field_id=3, name="age", field_type=IntegerType(), required=False),
-    )
-    
-    # Create table
-    try:
-        table = catalog.create_table(
-            identifier=table_id,
-            schema=schema,
-        )
-        print(f"  Created table: {table_id}")
-    except TableAlreadyExistsError:
-        print(f"  ! Table already exists: {table_id}")
-        _ = catalog.load_table(table_id)
-    
-    # List tables
-    tables = catalog.list_tables(namespace)
-    if table_name in [t[1] for t in tables]:
-        print("  Table appears in list")
-    else:
-        print(f"  Table not found in list: {tables}")
-        return False
-    
-    # Load table
-    try:
-        loaded_table = catalog.load_table(table_id)
-        print(f"  Loaded table: {loaded_table.name()}")
-        print(f"    Schema: {loaded_table.schema()}")
-        print(f"    Location: {loaded_table.location()}")
-    except NoSuchTableError:
-        print(f"  Failed to load table")
-        return False
-    
-    return True
-
-
-def test_cleanup(catalog, prefix):
-    """Test table and namespace deletion."""
-    print("Testing cleanup operations...")
-    namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
-    table_id = namespace + ("auth_test_table",)
-    
-    # Drop table
-    try:
-        catalog.drop_table(table_id)
-        print(f"  Dropped table: {table_id}")
-    except NoSuchTableError:
-        print(f"  ! Table already deleted: {table_id}")
-    
-    # Drop namespace
-    try:
-        catalog.drop_namespace(namespace)
-        print(f"  Dropped namespace: {namespace}")
-    except NoSuchNamespaceError:
-        print(f"  ! Namespace already deleted: {namespace}")
-    except Exception as e:
-        print(f"  ? Namespace drop error (may be expected): {e}")
-    
-    return True
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Test Iceberg REST Catalog with authentication")
-    parser.add_argument("--catalog-url", required=True, help="Iceberg REST Catalog URL")
-    parser.add_argument("--warehouse", default="s3://iceberg-test/", help="Warehouse location")
-    parser.add_argument("--prefix", required=True, help="Table bucket prefix")
-    parser.add_argument("--access-key", required=True, help="AWS Access Key ID")
-    parser.add_argument("--secret-key", required=True, help="AWS Secret Access Key")
-    parser.add_argument("--skip-cleanup", action="store_true", help="Skip cleanup at the end")
-    args = parser.parse_args()
-    
-    print(f"Connecting to Iceberg REST Catalog at: {args.catalog_url}")
-    print(f"Warehouse: {args.warehouse}")
-    print(f"Prefix: {args.prefix}")
-    print(f"Using authenticated access with key: {args.access_key}")
-    print()
-    
-    # Load the REST catalog with authentication
-    import time
-    max_retries = 10
-    catalog = None
-    for attempt in range(max_retries):
-        try:
-            catalog = load_catalog(
-                "rest",
-                **{
-                    "type": "rest",
-                    "uri": args.catalog_url,
-                    "warehouse": args.warehouse,
-                    "prefix": args.prefix,
-                    "s3.access-key-id": args.access_key,
-                    "s3.secret-access-key": args.secret_key,
-                }
-            )
-            print(f"Successfully connected to catalog on attempt {attempt + 1}")
-            break
-        except Exception as e:
-            if attempt < max_retries - 1:
-                print(f"  Attempt {attempt + 1} failed, retrying in 2s... ({e})")
-                time.sleep(2)
-            else:
-                print(f"  All {max_retries} attempts failed.")
-                raise e
-    
-    # Run tests
-    tests = [
-        ("Config Endpoint", lambda: test_config_endpoint(catalog)),
-        ("Namespace Operations", lambda: test_namespace_operations(catalog, args.prefix)),
-        ("Table Operations", lambda: test_table_operations(catalog, args.prefix)),
-    ]
-    
-    if not args.skip_cleanup:
-        tests.append(("Cleanup", lambda: test_cleanup(catalog, args.prefix)))
-    
-    passed = 0
-    failed = 0
-    
-    for name, test_fn in tests:
-        print(f"\n{'='*50}")
-        try:
-            if test_fn():
-                passed += 1
-                print(f"PASSED: {name}")
-            else:
-                failed += 1
-                print(f"FAILED: {name}")
-        except Exception as e:
-            failed += 1
-            print(f"ERROR in {name}: {e}")
-    
-    print(f"\n{'='*50}")
-    print(f"Results: {passed} passed, {failed} failed")
-    
-    return 0 if failed == 0 else 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())

test/s3tables/table-buckets/s3tables_integration_test.go

@@ -12,57 +12,33 @@ import (
 	"time"
 
 	cryptorand "crypto/rand"
+	"sync"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"flag"
-
 	"github.com/seaweedfs/seaweedfs/weed/command"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
 )
 
-// TestMain starts a single default weed mini cluster for the whole package and
-// tears it down after all tests have completed. Tests that require a different
-// cluster configuration (e.g. TestS3TablesCreateBucketIAMPolicy) start their
-// own cluster independently.
-func TestMain(m *testing.M) {
-	flag.Parse()
-	if testing.Short() {
-		// Tests self-skip with t.Skip when -short is set; no cluster needed.
-		os.Exit(m.Run())
-	}
-
-	// Create a temporary T-less context so we can use t.TempDir-equivalent.
-	testDir, err := os.MkdirTemp("", "seaweed-s3tables-shared-*")
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "SKIP: failed to create shared temp dir: %v\n", err)
-		os.Exit(0)
-	}
-
-	cluster, err := startMiniClusterInDir(testDir, nil)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "SKIP: failed to start shared weed mini cluster: %v\n", err)
-		os.RemoveAll(testDir)
-		os.Exit(0)
-	}
-	sharedCluster = cluster
-
-	code := m.Run()
-
-	sharedCluster.Stop()
-	os.RemoveAll(testDir)
-	os.Exit(code)
-}
+var (
+	miniClusterMutex sync.Mutex
+)
 
 func TestS3TablesIntegration(t *testing.T) {
 	if testing.Short() {
 		t.Skip("Skipping integration test in short mode")
 	}
 
-	// Re-use the shared cluster started by TestMain.
-	client := NewS3TablesClient(sharedCluster.s3Endpoint, testRegion, testAccessKey, testSecretKey)
+	// Create and start test cluster
+	cluster, err := startMiniCluster(t)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	// Create S3 Tables client
+	client := NewS3TablesClient(cluster.s3Endpoint, testRegion, testAccessKey, testSecretKey)
 
 	// Run test suite
 	t.Run("TableBucketLifecycle", func(t *testing.T) {
@@ -94,92 +70,6 @@ func TestS3TablesIntegration(t *testing.T) {
 	})
 }
 
-func TestS3TablesCreateBucketIAMPolicy(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping IAM integration test in short mode")
-	}
-
-	t.Setenv("AWS_ACCESS_KEY_ID", "env-admin")
-	t.Setenv("AWS_SECRET_ACCESS_KEY", "env-secret")
-
-	allowedBucket := "tables-allowed"
-	deniedBucket := "tables-denied"
-	iamConfigDir := t.TempDir()
-	iamConfigPath := filepath.Join(iamConfigDir, "iam_config.json")
-	iamConfig := fmt.Sprintf(`{
-  "sts": {
-    "tokenDuration": "1h",
-    "maxSessionLength": "12h",
-    "issuer": "seaweedfs-sts",
-    "signingKey": "%s"
-  },
-  "accounts": [
-    {
-      "id": "%s",
-      "displayName": "tables-integration"
-    }
-  ],
-  "identities": [
-    {
-      "name": "admin",
-      "credentials": [
-        {
-          "accessKey": "%s",
-          "secretKey": "%s"
-        }
-      ],
-      "account": {
-        "id": "%s",
-        "displayName": "tables-integration"
-      },
-      "policyNames": ["S3TablesBucketPolicy"]
-    }
-  ],
-  "policy": {
-    "defaultEffect": "Deny",
-    "storeType": "memory"
-  },
-  "policies": [
-    {
-      "name": "S3TablesBucketPolicy",
-      "document": {
-        "Version": "2012-10-17",
-        "Statement": [
-          {
-            "Effect": "Allow",
-            "Action": ["s3tables:CreateTableBucket"],
-            "Resource": [
-              "arn:aws:s3tables:*:*:bucket/%s",
-              "arn:aws:s3:::%s"
-            ]
-          }
-        ]
-      }
-    }
-  ]
-}`, testIAMSigningKey, testAccountID, testAccessKey, testSecretKey, testAccountID, allowedBucket, allowedBucket)
-	require.NoError(t, os.WriteFile(iamConfigPath, []byte(iamConfig), 0644))
-
-	cluster, err := startMiniClusterWithExtraArgs(t, []string{
-		"-s3.config=" + iamConfigPath,
-		"-s3.iam.config=" + iamConfigPath,
-	})
-	require.NoError(t, err, "failed to start cluster with IAM config")
-	defer cluster.Stop()
-
-	client := NewS3TablesClient(cluster.s3Endpoint, testRegion, testAccessKey, testSecretKey)
-
-	_, err = client.CreateTableBucket(deniedBucket, nil)
-	require.Error(t, err, "denied bucket creation should fail")
-	assert.Contains(t, err.Error(), "AccessDenied")
-
-	allowedResp, err := client.CreateTableBucket(allowedBucket, nil)
-	require.NoError(t, err, "allowed bucket creation should succeed")
-	defer func() {
-		_ = client.DeleteTableBucket(allowedResp.ARN)
-	}()
-}
-
 func testTableBucketLifecycle(t *testing.T, client *S3TablesClient) {
 	bucketName := "test-bucket-" + randomString(8)
 
@@ -618,12 +508,11 @@ func findAvailablePorts(n int) ([]int, error) {
 	return ports, nil
 }
 
-// startMiniClusterInDir starts a weed mini instance using testDir as the data
-// directory. It does not require a *testing.T so it can be called from TestMain.
-// extraArgs are appended to the default mini command flags.
-func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, error) {
-	// We need 10 unique ports: Master(2), Volume(2), Filer(2), S3(2), Admin(2)
-	ports, err := findAvailablePorts(10)
+// startMiniCluster starts a weed mini instance directly without exec
+func startMiniCluster(t *testing.T) (*TestCluster, error) {
+	// Find available ports
+	// We need 8 unique ports: Master(2), Volume(2), Filer(2), S3(2)
+	ports, err := findAvailablePorts(8)
 	if err != nil {
 		return nil, fmt.Errorf("failed to find available ports: %v", err)
 	}
@@ -636,8 +525,8 @@ func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, er
 	filerGrpcPort := ports[5]
 	s3Port := ports[6]
 	s3GrpcPort := ports[7]
-	adminPort := ports[8]
-	adminGrpcPort := ports[9]
+	// Create temporary directory for test data
+	testDir := t.TempDir()
 
 	// Ensure no configuration file from previous runs
 	configFile := filepath.Join(testDir, "mini.options")
@@ -648,6 +537,7 @@ func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, er
 
 	s3Endpoint := fmt.Sprintf("http://127.0.0.1:%d", s3Port)
 	cluster := &TestCluster{
+		t:          t,
 		dataDir:    testDir,
 		ctx:        ctx,
 		cancel:     cancel,
@@ -660,25 +550,22 @@ func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, er
 
 	// Create empty security.toml to disable JWT authentication in tests
 	securityToml := filepath.Join(testDir, "security.toml")
-	if err = os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644); err != nil {
+	err = os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
+	if err != nil {
 		cancel()
 		return nil, fmt.Errorf("failed to create security.toml: %v", err)
 	}
 
-	// Ensure AWS credentials are set (don't use t.Setenv here — we are in TestMain).
-	if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
-		os.Setenv("AWS_ACCESS_KEY_ID", "admin")
-	}
-	if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
-		os.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
-	}
-
 	// Start weed mini in a goroutine by calling the command directly
 	cluster.wg.Add(1)
 	go func() {
 		defer cluster.wg.Done()
 
-		// Save current directory and args, restore on exit.
+		// Protect global state mutation with a mutex
+		miniClusterMutex.Lock()
+		defer miniClusterMutex.Unlock()
+
+		// Save current directory and args
 		oldDir, _ := os.Getwd()
 		oldArgs := os.Args
 		defer func() {
@@ -686,24 +573,21 @@ func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, er
 			os.Args = oldArgs
 		}()
 
-		// Change to test directory so mini picks up security.toml.
+		// Change to test directory so mini picks up security.toml
 		os.Chdir(testDir)
 
-		baseArgs := []string{
+		// Configure args for mini command
+		os.Args = []string{
+			"weed",
 			"-dir=" + testDir,
-			"-master.dir=" + testDir,
 			"-master.port=" + strconv.Itoa(masterPort),
 			"-master.port.grpc=" + strconv.Itoa(masterGrpcPort),
 			"-volume.port=" + strconv.Itoa(volumePort),
 			"-volume.port.grpc=" + strconv.Itoa(volumeGrpcPort),
-			"-volume.port.public=" + strconv.Itoa(volumePort),
-			"-volume.publicUrl=127.0.0.1:" + strconv.Itoa(volumePort),
 			"-filer.port=" + strconv.Itoa(filerPort),
 			"-filer.port.grpc=" + strconv.Itoa(filerGrpcPort),
 			"-s3.port=" + strconv.Itoa(s3Port),
 			"-s3.port.grpc=" + strconv.Itoa(s3GrpcPort),
-			"-admin.port=" + strconv.Itoa(adminPort),
-			"-admin.port.grpc=" + strconv.Itoa(adminGrpcPort),
 			"-webdav.port=0",               // Disable WebDAV
 			"-admin.ui=false",              // Disable admin UI
 			"-master.volumeSizeLimitMB=32", // Small volumes for testing
@@ -711,10 +595,6 @@ func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, er
 			"-master.peers=none",     // Faster startup
 			"-s3.iam.readOnly=false", // Enable IAM write operations for tests
 		}
-		if len(extraArgs) > 0 {
-			baseArgs = append(baseArgs, extraArgs...)
-		}
-		os.Args = append([]string{"weed"}, baseArgs...)
 
 		// Suppress most logging during tests
 		glog.MaxSize = 1024 * 1024
@@ -733,34 +613,14 @@ func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, er
 	}()
 
 	// Wait for S3 service to be ready
-	if err = waitForS3Ready(cluster.s3Endpoint, 30*time.Second); err != nil {
+	err = waitForS3Ready(cluster.s3Endpoint, 30*time.Second)
+	if err != nil {
 		cancel()
 		return nil, fmt.Errorf("S3 service failed to start: %v", err)
 	}
 
 	cluster.isRunning = true
-	return cluster, nil
-}
 
-// startMiniClusterWithExtraArgs starts a weed mini instance for a single test.
-// It uses t.TempDir() for data isolation and t.Setenv for credential scoping.
-func startMiniClusterWithExtraArgs(t *testing.T, extraArgs []string) (*TestCluster, error) {
-	t.Helper()
-	testDir := t.TempDir()
-
-	// Scope credentials to the test so they are restored after test completion.
-	if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
-		t.Setenv("AWS_ACCESS_KEY_ID", "admin")
-	}
-	if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
-		t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
-	}
-
-	cluster, err := startMiniClusterInDir(testDir, extraArgs)
-	if err != nil {
-		return nil, err
-	}
-	cluster.t = t
 	t.Logf("Test cluster started successfully at %s", cluster.s3Endpoint)
 	return cluster, nil
 }
@@ -786,15 +646,22 @@ func (c *TestCluster) Stop() {
 	case <-done:
 		// Goroutine finished
 	case <-timer.C:
-		// Timeout - goroutine doesn't respond to context cancel.
-		// This may indicate the mini cluster didn't shut down cleanly.
-		if c.t != nil {
-			c.t.Log("Warning: Test cluster shutdown timed out after 2 seconds")
-		} else {
-			fmt.Println("Warning: Test cluster shutdown timed out after 2 seconds")
-		}
+		// Timeout - goroutine doesn't respond to context cancel
+		// This may indicate the mini cluster didn't shut down cleanly
+		c.t.Log("Warning: Test cluster shutdown timed out after 2 seconds")
 	}
 
+	// Reset the global cmdMini flags to prevent state leakage to other tests
+	for _, cmd := range command.Commands {
+		if cmd.Name() == "mini" {
+			// Reset flags to defaults
+			cmd.Flag.VisitAll(func(f *flag.Flag) {
+				// Reset to default value
+				f.Value.Set(f.DefValue)
+			})
+			break
+		}
+	}
 }
 
 // waitForS3Ready waits for the S3 service to be ready

test/s3tables/table-buckets/setup.go

@@ -8,11 +8,6 @@ import (
 	"time"
 )
 
-// sharedCluster is the single default TestCluster shared across all tests
-// that do not require a specialised cluster configuration.
-// It is initialised by TestMain and must not be modified by individual tests.
-var sharedCluster *TestCluster
-
 // TestCluster manages the weed mini instance for integration testing
 type TestCluster struct {
 	t          *testing.T
@@ -51,9 +46,8 @@ func NewS3TablesClient(endpoint, region, accessKey, secretKey string) *S3TablesC
 
 // Test configuration constants
 const (
-	testRegion        = "us-west-2"
-	testAccessKey     = "admin"
-	testSecretKey     = "admin"
-	testAccountID     = "111122223333"
-	testIAMSigningKey = "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz"
+	testRegion    = "us-west-2"
+	testAccessKey = "admin"
+	testSecretKey = "admin"
+	testAccountID = "111122223333"
 )

test/volume_server/DEV_PLAN.md

@@ -3,6 +3,12 @@
 ## Goal
 Create a Go integration test suite under `test/volume_server` that validates **drop-in behavior parity** for the Volume Server HTTP and gRPC APIs, so a Rust rewrite can be verified against the current Go behavior.
 
+## Current Program Focus (2026-02-16)
+- Primary execution focus has shifted to implementing native Rust volume-server parity.
+- This integration suite is now the parity gate for Rust implementation work.
+- New tests should be added only when native Rust implementation reveals uncovered Go behavior that is not yet captured.
+- Rust implementation roadmap lives in `rust/volume_server/DEV_PLAN.md`.
+
 ## Hard Requirements
 - Tests live under `test/volume_server`.
 - Tests are written in Go.
@@ -1127,3 +1133,202 @@ Update this section during implementation:
 - Profiles covered: P1.
 - Gaps introduced/remaining: `499` cancellation and transport-interruption branches remain pending.
 - Commit: `d1e5f390a`
+
+- Date: 2026-02-15
+- Change: Added Rust volume-server migration bootstrap wiring.
+- APIs covered: Harness now supports Go master + selectable volume binary (`VOLUME_SERVER_IMPL=rust` or `VOLUME_SERVER_BINARY`), with Rust-mode smoke coverage for representative HTTP/gRPC integration tests.
+- Profiles covered: P1 smoke and default Go matrix unchanged.
+- Gaps introduced/remaining: native Rust handler implementations are tracked in `rust/volume_server/DEV_PLAN.md`; current Rust mode is a compatibility launcher phase.
+- Commit: `7beab85c2`, `880c2e1da`, `63d08e8a9`, `d402573ea`
+
+- Date: 2026-02-15
+- Change: Validated Rust-mode compatibility path with full integration suites.
+- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` suites executed successfully with `VOLUME_SERVER_IMPL=rust`.
+- Profiles covered: existing integration matrix in Rust launcher mode.
+- Gaps introduced/remaining: native Rust endpoint and storage/RPC implementation remains pending; current mode delegates execution to Go volume server.
+- Commit: `6ce4d7ede`
+
+- Date: 2026-02-15
+- Change: Added Rust proxy-supervision launcher mode and validated all volume-server integration suites in proxy mode.
+- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` suites executed successfully with `VOLUME_SERVER_IMPL=rust` and `VOLUME_SERVER_RUST_MODE=proxy`.
+- Profiles covered: existing integration matrix in Rust proxy launcher mode.
+- Gaps introduced/remaining: native Rust endpoint/storage/RPC logic remains pending; current proxy mode still delegates backend handlers to Go.
+- Commit: `a7f50d23b`
+
+- Date: 2026-02-15
+- Change: Expanded GitHub CI Rust smoke job to run both launcher modes (`exec` and `proxy`).
+- APIs covered: representative HTTP/gRPC smoke flows now validated in CI for both Rust launcher execution paths.
+- Profiles covered: CI smoke profile for P1 representative calls.
+- Gaps introduced/remaining: CI still runs Rust mode as smoke scope; full-suite Rust-mode CI remains optional due runtime cost.
+- Commit: `548b3d9a3`
+
+- Date: 2026-02-15
+- Change: Added JWT UI override integration coverage with explicit `access.ui=true` profile support.
+- APIs covered: `/ui/index.html` behavior under JWT profile now verifies both default auth-gated path (`401`) and explicit UI-enable override path (`200` with rendered UI).
+- Profiles covered: P3 and P3+`access.ui=true`.
+- Gaps introduced/remaining: UI exposure behavior under JWT profiles is now covered for both secured and override branches.
+- Commit: `de974c05d`
+
+- Date: 2026-02-15
+- Change: Added oversized upload integration coverage using explicit `-fileSizeLimitMB` profile configuration.
+- APIs covered: HTTP write path now verifies payloads larger than configured file-size limit are rejected (`400`) with limit-related error context.
+- Profiles covered: P1-derived profile with `FileSizeLimitMB=1`.
+- Gaps introduced/remaining: file-size limit rejection branch is now covered; replicate-write failure and cancellation (`499`) branches remain pending.
+- Commit: `4d61cbdee`
+
+- Date: 2026-02-15
+- Change: Added gRPC EC-only metadata-read unsupported-path coverage.
+- APIs covered: `ReadNeedleMeta` now verifies the explicit unsupported branch when a volume is unmounted and only EC shards are mounted.
+- Profiles covered: P1 with EC shard generation/mount lifecycle setup.
+- Gaps introduced/remaining: low-level read metadata fault-injection and transport interruption branches remain pending.
+- Commit: `37bf9b5eb`
+
+- Date: 2026-02-15
+- Change: Added deterministic replicated-write failure coverage for unmet replication requirements.
+- APIs covered: HTTP write path now verifies replication error handling (`500`) and no-local-commit outcome (`404` on follow-up read) when volume replication is configured (`001`) but replica set cannot be satisfied.
+- Profiles covered: P1-derived single-node profile with per-volume replication override.
+- Gaps introduced/remaining: replicated-write failure branch is now covered; cancellation (`499`) and deeper transport fault-injection branches remain pending.
+- Commit: `4835d3443`
+
+- Date: 2026-02-15
+- Change: Added EC-backed `BatchDelete` positive-path integration coverage.
+- APIs covered: `BatchDelete` now validates successful EC-needle deletion (`202`) with post-delete `VolumeEcShardRead` deleted-marker verification (`IsDeleted=true`).
+- Profiles covered: P1 with EC shard generate/mount lifecycle; includes test-local gRPC offset proxy for EC internal dial-path parity.
+- Gaps introduced/remaining: EC `BatchDelete` success branch is now covered; deeper distributed failure-injection permutations remain pending.
+- Commit: `1bb40b6bc`
+
+- Date: 2026-02-15
+- Change: Expanded HTTP conditional-header matrix for `HEAD` requests with combined header scenarios.
+- APIs covered: `HEAD` now verifies `If-Modified-Since` precedence over mismatched `If-None-Match`, and `If-None-Match` precedence when IMS is stale but ETag matches.
+- Profiles covered: P1.
+- Gaps introduced/remaining: combined conditional-header precedence paths for `HEAD`/`GET` are now explicitly covered.
+- Commit: `ed23e290f`
+
+- Date: 2026-02-15
+- Change: Added multi-volume success-stream coverage for `ReadAllNeedles`.
+- APIs covered: `ReadAllNeedles` now verifies happy-path streaming across two existing volumes in one request, with per-volume payload validation.
+- Profiles covered: P1.
+- Gaps introduced/remaining: multi-volume stream boundary coverage is now present for successful reads; transport interruption branches remain pending.
+- Commit: `ab95a6ef1`
+
+- Date: 2026-02-15
+- Change: Expanded split-port unsupported HTTP method matrix with `MKCOL` coverage.
+- APIs covered: admin/public parity for `MKCOL` (`400` on admin, passthrough `200` on public) with post-call data-integrity verification.
+- Profiles covered: P2.
+- Gaps introduced/remaining: unsupported-method sampling now covers `PATCH`, `TRACE`, `PROPFIND`, `CONNECT`, and `MKCOL`; cancellation/transport fault paths remain the main unaddressed area.
+- Commit: `2ab30900d`
+
+- Date: 2026-02-15
+- Change: Hardened framework port allocation to keep volume ports within `10000..55535`.
+- APIs covered: test harness now guarantees valid `admin+10000` gRPC offset ranges, eliminating EC batch-delete flake scenarios that rely on offset dialing.
+- Profiles covered: all profiles using shared framework port allocation.
+- Gaps introduced/remaining: no new API gaps; transport cancellation/fault-injection branches remain the primary uncovered area.
+- Commit: `90e82b15c`
+
+- Date: 2026-02-15
+- Change: Re-validated full Rust proxy-mode integration suite after latest HTTP/gRPC coverage additions and framework hardening.
+- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` packages.
+- Profiles covered: existing matrix in Rust proxy launcher mode.
+- Gaps introduced/remaining: native Rust endpoint/storage/RPC implementation remains pending (current Rust mode still delegates to Go backend handlers).
+- Commit: `6bb9d8bac`
+
+- Date: 2026-02-15
+- Change: Added tail sender stream-cancellation interruption coverage.
+- APIs covered: `VolumeTailSender` now verifies client-side context cancellation behavior (`codes.Canceled`) after stream start/heartbeat.
+- Profiles covered: P1.
+- Gaps introduced/remaining: tail transport interruption coverage is now partially closed; receiver-side interruption and deeper network fault injection remain pending.
+- Commit: `27a80f760`
+
+- Date: 2026-02-15
+- Change: Added receiver-side tail stream interruption coverage for unavailable source node.
+- APIs covered: `VolumeTailReceiver` now verifies source connect/dial failure branch when `SourceVolumeServer` is unreachable.
+- Profiles covered: P1.
+- Gaps introduced/remaining: both sender and receiver transport interruption branches are now covered at integration level; deeper injected network faults remain pending.
+- Commit: `a5864c3eb`
+
+- Date: 2026-02-15
+- Change: Expanded query CSV matrix with explicit CSV-payload parity coverage.
+- APIs covered: `Query` now verifies current CSV-input behavior (no streamed rows / immediate EOF) even when source needle content is valid CSV text.
+- Profiles covered: P1.
+- Gaps introduced/remaining: CSV parsing/selection implementation still absent in current server behavior; integration parity now locks the current no-output semantics for both JSON and CSV payload shapes.
+- Commit: `a12dd5f8d`
+
+- Date: 2026-02-15
+- Change: Expanded ping unreachable-target matrix with volume-server target coverage.
+- APIs covered: `Ping` now explicitly verifies unreachable-target error wrapping for `target_type=volumeServer`, alongside existing master/filer variants.
+- Profiles covered: P1.
+- Gaps introduced/remaining: ping target-type matrix is covered for success and unreachable branches across master/filer/volume-server.
+- Commit: `b9fbb85af`
+
+- Date: 2026-02-15
+- Change: Expanded deleted-read HTTP matrix with `HEAD` parity coverage.
+- APIs covered: `HEAD` with `readDeleted=true` now validates current behavior (`200`, empty body, payload-size `Content-Length`) after delete.
+- Profiles covered: P1.
+- Gaps introduced/remaining: deleted-read parity now covers both `GET` and `HEAD` semantics on local-volume path.
+- Commit: `cc80ad364`
+
+- Date: 2026-02-16
+- Change: Shifted planning priority to native Rust implementation parity.
+- APIs covered: no new API additions in this plan entry; integration suite remains the validation gate.
+- Profiles covered: unchanged.
+- Gaps introduced/remaining: primary remaining gap is native Rust handler/storage/RPC implementation replacing Go backend delegation.
+- Commit: `14c863dbf`
+
+- Date: 2026-02-16
+- Change: Added Rust launcher `native` mode bootstrap, made it the default Rust launcher mode, and expanded CI Rust smoke matrix coverage to include `native`.
+- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` packages re-validated in `VOLUME_SERVER_RUST_MODE=native`; default Rust launcher path (`VOLUME_SERVER_IMPL=rust`) smoke-validated for HTTP and gRPC control tests.
+- Profiles covered: existing HTTP/gRPC matrix in native launcher mode; default-mode smoke checks on P1 control flows.
+- Gaps introduced/remaining: native Rust API/storage/RPC handlers still pending; current `native` mode remains a delegation bootstrap while parity replacement proceeds.
+- Commit: `70ddbee37`, `61befd10f`, `2e65966c0`
+
+- Date: 2026-02-16
+- Change: Implemented first native Rust HTTP control handlers in `native` mode for `/status` and `/healthz` (admin listener path), with proxy fallback retained for remaining endpoints.
+- APIs covered: `/status` and `/healthz` now served by Rust-native code path in `VOLUME_SERVER_RUST_MODE=native`; full HTTP and gRPC integration suites re-validated.
+- Profiles covered: full existing HTTP/gRPC matrix under native mode.
+- Gaps introduced/remaining: gRPC control/data APIs and most HTTP data/static/auth paths remain delegated and require incremental native replacement.
+- Commit: `7e6e0261a`
+
+- Date: 2026-02-16
+- Change: Implemented native Rust HTTP `OPTIONS` handling for admin/public listeners in `VOLUME_SERVER_RUST_MODE=native`.
+- APIs covered: admin/public `OPTIONS` method allow-list and CORS origin handling now served from Rust-native path; full HTTP and gRPC packages re-validated.
+- Profiles covered: P2 split-port CORS/OPTIONS profiles plus full existing matrix in native mode.
+- Gaps introduced/remaining: static/UI/auth and data-path HTTP handlers, plus gRPC control/data/stream handlers, remain delegated and pending native replacement.
+- Commit: `fbff2cb39`
+
+- Date: 2026-02-16
+- Change: Implemented broader native Rust HTTP surface in `native` mode:
+  - `/ui/index.html` auth-gating parity (JWT + access.ui handling)
+  - static asset paths (`/favicon.ico`, `/seaweedfsstatic/*`)
+  - public non-read method no-op parity (`200`) and admin unsupported-method rejection parity (`400`)
+- APIs covered: UI/static/admin/public control-surface behavior now served by Rust-native path; full `/test/volume_server/http` and `/test/volume_server/grpc` packages re-validated.
+- Profiles covered: P1/P2/P3 variants touched by UI/static/CORS/method behavior, plus full matrix in native mode.
+- Gaps introduced/remaining: HTTP data-path handlers and all gRPC handlers remain delegated and still require native replacement.
+- Commit: `23e4497b2`
+
+- Date: 2026-02-16
+- Change: Improved native Rust control-path parity for health/status and request-target parsing.
+- APIs covered: native `/healthz` now mirrors backend service health transitions (`200`/`503`) and native route matching now handles absolute-form HTTP request targets before path dispatch.
+- Profiles covered: full existing HTTP/gRPC integration matrix in native mode (`VOLUME_SERVER_IMPL=rust`, `VOLUME_SERVER_RUST_MODE=native`).
+- Gaps introduced/remaining: core HTTP data handlers and all gRPC RPC handlers are still delegated and remain the primary native implementation gap.
+- Commit: `d6ff6ed6d`
+
+- Date: 2026-02-16
+- Change: Added Rust-native malformed fid-route validation ahead of delegated data handlers.
+- APIs covered: GET/HEAD/POST/PUT fid-shaped paths now return native `400` for invalid vid/fid tokens (including invalid-read and invalid-write path variants) while valid routes continue through delegated storage handlers.
+- Profiles covered: full existing HTTP/gRPC integration matrix in native mode, plus targeted invalid-path parity runs.
+- Gaps introduced/remaining: data-path success branches and all gRPC handler bodies remain delegated; native replacement work continues on those core execution paths.
+- Commit: `1ce0174b2`
+
+- Date: 2026-02-16
+- Change: Added Rust-native write error prevalidation for delegated fid routes.
+- APIs covered: native path now rejects malformed multipart boundary requests, `Content-MD5` writes, and over-limit `Content-Length` writes (using `-fileSizeLimitMB`) with `400` parity behavior before delegation.
+- Profiles covered: P1 default and P1 with explicit file-size limit profile, plus full existing HTTP/gRPC native-mode matrix revalidation.
+- Gaps introduced/remaining: positive write/read/delete data handlers and all gRPC handler bodies are still delegated; native implementation continues incrementally.
+- Commit: `94cefd6f4`
+
+- Date: 2026-02-16
+- Change: Broadened Rust-native malformed-fid handling to slash-form read/write URL shapes with reserved-route exclusions.
+- APIs covered: `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` malformed cases now hit native `400` validation path, while `/status`, `/healthz`, `/ui/index.html`, `/stats/*`, and static asset routes remain excluded from fid parsing.
+- Profiles covered: full existing HTTP/gRPC native-mode matrix, plus targeted admin/read-path malformed-slash validation run.
+- Gaps introduced/remaining: successful data-path read/write/delete handlers and all gRPC method bodies remain delegated and are still the primary native implementation gap.
+- Commit: `e66983e9b`

test/volume_server/Makefile

@@ -1,7 +1,10 @@
-.PHONY: test-volume-server test-volume-server-short
+.PHONY: test-volume-server test-volume-server-short test-volume-server-rust-smoke
 
 test-volume-server:
 	go test ./test/volume_server/... -v
 
 test-volume-server-short:
 	go test ./test/volume_server/... -short -v
+
+test-volume-server-rust-smoke:
+	VOLUME_SERVER_IMPL=rust go test ./test/volume_server/http ./test/volume_server/grpc -run 'TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip|TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus' -v

test/volume_server/README.md

@@ -15,6 +15,10 @@ If a `weed` binary is not found, the harness will build one automatically.
 ## Optional environment variables
 
 - `WEED_BINARY`: explicit path to the `weed` executable (disables auto-build).
+- `VOLUME_SERVER_IMPL`: select volume-server implementation (`go` default, `rust` for Rust launcher mode).
+- `VOLUME_SERVER_BINARY`: explicit path to the volume-server executable (overrides `VOLUME_SERVER_IMPL`).
+- `VOLUME_SERVER_RUST_MODE`: Rust launcher mode (`native` default, `exec` for direct Go delegation, `proxy` for Rust front proxy + Go backend).
+- `VOLUME_SERVER_RUST_REBUILD=1`: force rebuild of Rust volume-server binary in Rust mode.
 - `VOLUME_SERVER_IT_KEEP_LOGS=1`: keep temporary test directories and process logs.
 
 ## Current scope (Phase 0)
@@ -24,4 +28,4 @@ If a `weed` binary is not found, the harness will build one automatically.
 - Initial HTTP admin endpoint checks
 - Initial gRPC state/status checks
 
-More API coverage is tracked in `/Users/chris/dev/seaweedfs2/test/volume_server/DEV_PLAN.md`.
+More API coverage is tracked in `test/volume_server/DEV_PLAN.md`.

test/volume_server/framework/cluster.go

@@ -32,11 +32,12 @@ type Cluster struct {
 	testingTB testing.TB
 	profile   matrix.Profile
 
-	weedBinary string
-	baseDir    string
-	configDir  string
-	logsDir    string
-	keepLogs   bool
+	weedBinary   string
+	volumeBinary string
+	baseDir      string
+	configDir    string
+	logsDir      string
+	keepLogs     bool
 
 	masterPort     int
 	masterGrpcPort int
@@ -54,9 +55,9 @@ type Cluster struct {
 func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
 	t.Helper()
 
-	weedBinary, err := FindOrBuildWeedBinary()
+	weedBinary, volumeBinary, err := FindOrBuildServerBinaries()
 	if err != nil {
-		t.Fatalf("resolve weed binary: %v", err)
+		t.Fatalf("resolve server binaries: %v", err)
 	}
 
 	baseDir, keepLogs, err := newWorkDir()
@@ -92,6 +93,7 @@ func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
 		testingTB:      t,
 		profile:        profile,
 		weedBinary:     weedBinary,
+		volumeBinary:   volumeBinary,
 		baseDir:        baseDir,
 		configDir:      configDir,
 		logsDir:        logsDir,
@@ -206,8 +208,11 @@ func (c *Cluster) startVolume(dataDir string) error {
 	if c.profile.InflightDownloadTimeout > 0 {
 		args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
 	}
+	if c.profile.FileSizeLimitMB > 0 {
+		args = append(args, "-fileSizeLimitMB="+strconv.Itoa(c.profile.FileSizeLimitMB))
+	}
 
-	c.volumeCmd = exec.Command(c.weedBinary, args...)
+	c.volumeCmd = exec.Command(c.volumeBinary, args...)
 	c.volumeCmd.Dir = c.baseDir
 	c.volumeCmd.Stdout = logFile
 	c.volumeCmd.Stderr = logFile
@@ -264,18 +269,46 @@ func stopProcess(cmd *exec.Cmd) {
 }
 
 func allocatePorts(count int) ([]int, error) {
+	const minPort = 10000
+	const maxPort = 55535
+	const host = "127.0.0.1"
+	rangeSize := maxPort - minPort + 1
+
 	listeners := make([]net.Listener, 0, count)
 	ports := make([]int, 0, count)
+	seen := make(map[int]struct{}, count)
+
+	closeAll := func() {
+		for _, ll := range listeners {
+			_ = ll.Close()
+		}
+	}
+
+	startOffset := int(time.Now().UnixNano() % int64(rangeSize))
 	for i := 0; i < count; i++ {
-		l, err := net.Listen("tcp", "127.0.0.1:0")
-		if err != nil {
-			for _, ll := range listeners {
-				_ = ll.Close()
+		found := false
+		for offset := 0; offset < rangeSize; offset++ {
+			port := minPort + (startOffset+offset)%rangeSize
+			if _, exists := seen[port]; exists {
+				continue
 			}
-			return nil, err
+
+			l, err := net.Listen("tcp", net.JoinHostPort(host, strconv.Itoa(port)))
+			if err != nil {
+				continue
+			}
+
+			seen[port] = struct{}{}
+			listeners = append(listeners, l)
+			ports = append(ports, port)
+			found = true
+			startOffset = (startOffset + offset + 1) % rangeSize
+			break
+		}
+		if !found {
+			closeAll()
+			return nil, fmt.Errorf("unable to allocate %d ports within range [%d,%d]", count, minPort, maxPort)
 		}
-		listeners = append(listeners, l)
-		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
 	}
 	for _, l := range listeners {
 		_ = l.Close()
@@ -326,6 +359,13 @@ func writeSecurityConfig(configDir string, profile matrix.Profile) error {
 		b.WriteString("\"\n")
 		b.WriteString("expires_after_seconds = 60\n")
 	}
+	if profile.AccessUI {
+		if b.Len() > 0 {
+			b.WriteString("\n")
+		}
+		b.WriteString("[access]\n")
+		b.WriteString("ui = true\n")
+	}
 	if b.Len() == 0 {
 		b.WriteString("# optional security config generated for integration tests\n")
 	}
@@ -341,17 +381,12 @@ func FindOrBuildWeedBinary() (string, error) {
 		return "", fmt.Errorf("WEED_BINARY is set but not executable: %s", fromEnv)
 	}
 
-	repoRoot := ""
-	if _, file, _, ok := runtime.Caller(0); ok {
-		repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
-		candidate := filepath.Join(repoRoot, "weed", "weed")
-		if isExecutableFile(candidate) {
-			return candidate, nil
-		}
+	repoRoot, err := detectRepoRoot()
+	if err != nil {
+		return "", err
 	}
-
-	if repoRoot == "" {
-		return "", errors.New("unable to detect repository root")
+	if candidate := filepath.Join(repoRoot, "weed", "weed"); isExecutableFile(candidate) {
+		return candidate, nil
 	}
 
 	binDir := filepath.Join(os.TempDir(), "seaweedfs_volume_server_it_bin")
@@ -377,6 +412,82 @@ func FindOrBuildWeedBinary() (string, error) {
 	return binPath, nil
 }
 
+// FindOrBuildServerBinaries returns master and volume executables.
+// Master always runs from the Go weed binary; volume can be switched via env.
+func FindOrBuildServerBinaries() (masterBinary string, volumeBinary string, err error) {
+	masterBinary, err = FindOrBuildWeedBinary()
+	if err != nil {
+		return "", "", err
+	}
+	volumeBinary, err = FindOrBuildVolumeServerBinary(masterBinary)
+	if err != nil {
+		return "", "", err
+	}
+	return masterBinary, volumeBinary, nil
+}
+
+// FindOrBuildVolumeServerBinary resolves the executable used for volume-server processes.
+//
+// Behavior:
+//   - `VOLUME_SERVER_BINARY=/path/to/bin`: use explicit executable path.
+//   - `VOLUME_SERVER_IMPL=rust`: build/use Rust volume server launcher.
+//   - default: use the same Go `weed` binary.
+func FindOrBuildVolumeServerBinary(defaultBinary string) (string, error) {
+	if fromEnv := os.Getenv("VOLUME_SERVER_BINARY"); fromEnv != "" {
+		if isExecutableFile(fromEnv) {
+			return fromEnv, nil
+		}
+		return "", fmt.Errorf("VOLUME_SERVER_BINARY is set but not executable: %s", fromEnv)
+	}
+
+	impl := strings.ToLower(strings.TrimSpace(os.Getenv("VOLUME_SERVER_IMPL")))
+	if impl == "" || impl == "go" {
+		return defaultBinary, nil
+	}
+	if impl != "rust" {
+		return "", fmt.Errorf("unsupported VOLUME_SERVER_IMPL %q (supported: go, rust)", impl)
+	}
+
+	repoRoot, err := detectRepoRoot()
+	if err != nil {
+		return "", err
+	}
+	return FindOrBuildRustVolumeServerBinary(repoRoot)
+}
+
+// FindOrBuildRustVolumeServerBinary builds the Rust volume server launcher when needed.
+func FindOrBuildRustVolumeServerBinary(repoRoot string) (string, error) {
+	manifestPath := filepath.Join(repoRoot, "rust", "volume_server", "Cargo.toml")
+	if _, err := os.Stat(manifestPath); err != nil {
+		return "", fmt.Errorf("rust volume server manifest not found at %s: %w", manifestPath, err)
+	}
+
+	targetDir := filepath.Join(os.TempDir(), "seaweedfs_volume_server_it_rust_target")
+	binPath := filepath.Join(targetDir, "release", "weed-volume-rs")
+	if isExecutableFile(binPath) && os.Getenv("VOLUME_SERVER_RUST_REBUILD") != "1" {
+		return binPath, nil
+	}
+
+	cmd := exec.Command("cargo", "build", "--release", "--manifest-path", manifestPath, "--target-dir", targetDir)
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+	if err := cmd.Run(); err != nil {
+		return "", fmt.Errorf("build rust volume server binary: %w\n%s", err, out.String())
+	}
+	if !isExecutableFile(binPath) {
+		return "", fmt.Errorf("built rust volume server binary is not executable: %s", binPath)
+	}
+	return binPath, nil
+}
+
+func detectRepoRoot() (string, error) {
+	if _, file, _, ok := runtime.Caller(0); ok {
+		return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..")), nil
+	}
+	return "", errors.New("unable to detect repository root")
+}
+
 func isExecutableFile(path string) bool {
 	info, err := os.Stat(path)
 	if err != nil || info.IsDir() {

test/volume_server/framework/cluster_dual.go

@@ -17,11 +17,12 @@ type DualVolumeCluster struct {
 	testingTB testing.TB
 	profile   matrix.Profile
 
-	weedBinary string
-	baseDir    string
-	configDir  string
-	logsDir    string
-	keepLogs   bool
+	weedBinary   string
+	volumeBinary string
+	baseDir      string
+	configDir    string
+	logsDir      string
+	keepLogs     bool
 
 	masterPort     int
 	masterGrpcPort int
@@ -43,9 +44,9 @@ type DualVolumeCluster struct {
 func StartDualVolumeCluster(t testing.TB, profile matrix.Profile) *DualVolumeCluster {
 	t.Helper()
 
-	weedBinary, err := FindOrBuildWeedBinary()
+	weedBinary, volumeBinary, err := FindOrBuildServerBinaries()
 	if err != nil {
-		t.Fatalf("resolve weed binary: %v", err)
+		t.Fatalf("resolve server binaries: %v", err)
 	}
 
 	baseDir, keepLogs, err := newWorkDir()
@@ -82,6 +83,7 @@ func StartDualVolumeCluster(t testing.TB, profile matrix.Profile) *DualVolumeClu
 		testingTB:       t,
 		profile:         profile,
 		weedBinary:      weedBinary,
+		volumeBinary:    volumeBinary,
 		baseDir:         baseDir,
 		configDir:       configDir,
 		logsDir:         logsDir,
@@ -227,7 +229,7 @@ func (c *DualVolumeCluster) startVolume(index int, dataDir string) error {
 		args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
 	}
 
-	cmd := exec.Command(c.weedBinary, args...)
+	cmd := exec.Command(c.volumeBinary, args...)
 	cmd.Dir = c.baseDir
 	cmd.Stdout = logFile
 	cmd.Stderr = logFile

test/volume_server/framework/volume_fixture.go

@@ -14,18 +14,26 @@ import (
 
 func AllocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string) {
 	t.Helper()
+	AllocateVolumeWithReplication(t, client, volumeID, collection, "000")
+}
+
+func AllocateVolumeWithReplication(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection, replication string) {
+	t.Helper()
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
+	if replication == "" {
+		replication = "000"
+	}
 
 	_, err := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{
 		VolumeId:    volumeID,
 		Collection:  collection,
-		Replication: "000",
+		Replication: replication,
 		Version:     uint32(needle.GetCurrentVersion()),
 	})
 	if err != nil {
-		t.Fatalf("allocate volume %d: %v", volumeID, err)
+		t.Fatalf("allocate volume %d (replication=%s): %v", volumeID, replication, err)
 	}
 }
 

test/volume_server/grpc/admin_extra_test.go

@@ -388,6 +388,17 @@ func TestPingUnknownAndUnreachableTargetPaths(t *testing.T) {
 	if !strings.Contains(err.Error(), "ping filer") {
 		t.Fatalf("Ping filer unreachable error mismatch: %v", err)
 	}
+
+	_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
+		TargetType: cluster.VolumeServerType,
+		Target:     "127.0.0.1:1.1",
+	})
+	if err == nil {
+		t.Fatalf("Ping volume server target should fail when target is unreachable")
+	}
+	if !strings.Contains(err.Error(), "ping volumeServer") {
+		t.Fatalf("Ping volume server unreachable error mismatch: %v", err)
+	}
 }
 
 func TestPingMasterTargetSuccess(t *testing.T) {

test/volume_server/grpc/batch_delete_test.go

@@ -3,7 +3,10 @@ package volume_server_grpc_test
 import (
 	"bytes"
 	"context"
+	"io"
+	"net"
 	"net/http"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -262,3 +265,141 @@ func TestBatchDeleteRejectsChunkManifestNeedles(t *testing.T) {
 		t.Fatalf("chunk manifest should not be deleted by BatchDelete reject path, got %d", readResp.StatusCode)
 	}
 }
+
+func TestBatchDeleteEcNeedleSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	if stopProxy := maybeStartGrpcOffsetProxy(t, cluster.VolumeAdminAddress(), cluster.VolumeGRPCAddress()); stopProxy != nil {
+		t.Cleanup(stopProxy)
+	}
+
+	const volumeID = uint32(34)
+	const needleID = uint64(930001)
+	const cookie = uint32(0x6677BBCC)
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), fid, []byte("batch-delete-ec-success"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := client.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	_, err = client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount failed: %v", err)
+	}
+
+	deleteResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{fid}})
+	if err != nil {
+		t.Fatalf("BatchDelete EC needle should return response, got grpc error: %v", err)
+	}
+	if len(deleteResp.GetResults()) != 1 {
+		t.Fatalf("BatchDelete EC needle expected one result, got %d", len(deleteResp.GetResults()))
+	}
+	if deleteResp.GetResults()[0].GetStatus() != http.StatusAccepted {
+		t.Fatalf("BatchDelete EC needle expected status 202, got %d error=%q", deleteResp.GetResults()[0].GetStatus(), deleteResp.GetResults()[0].GetError())
+	}
+	if deleteResp.GetResults()[0].GetSize() == 0 {
+		t.Fatalf("BatchDelete EC needle expected non-zero deleted size")
+	}
+
+	deletedStream, err := client.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
+		VolumeId: volumeID,
+		ShardId:  0,
+		FileKey:  needleID,
+		Offset:   0,
+		Size:     1,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardRead deleted-check start failed: %v", err)
+	}
+	deletedMsg, err := deletedStream.Recv()
+	if err != nil {
+		t.Fatalf("VolumeEcShardRead deleted-check recv failed: %v", err)
+	}
+	if !deletedMsg.GetIsDeleted() {
+		t.Fatalf("VolumeEcShardRead expected IsDeleted=true after EC batch delete")
+	}
+}
+
+func maybeStartGrpcOffsetProxy(t testing.TB, adminAddr, actualGrpcAddr string) func() {
+	t.Helper()
+
+	host, portText, err := net.SplitHostPort(adminAddr)
+	if err != nil {
+		t.Fatalf("split admin address %q: %v", adminAddr, err)
+	}
+	port, err := strconv.Atoi(portText)
+	if err != nil {
+		t.Fatalf("parse admin port %q: %v", portText, err)
+	}
+
+	expectedGrpcAddr := net.JoinHostPort(host, strconv.Itoa(port+10000))
+	if expectedGrpcAddr == actualGrpcAddr {
+		return nil
+	}
+
+	listener, err := net.Listen("tcp", expectedGrpcAddr)
+	if err != nil {
+		t.Fatalf("listen grpc-offset proxy %s -> %s: %v", expectedGrpcAddr, actualGrpcAddr, err)
+	}
+
+	stopped := make(chan struct{})
+	go func() {
+		for {
+			conn, acceptErr := listener.Accept()
+			if acceptErr != nil {
+				select {
+				case <-stopped:
+					return
+				default:
+					continue
+				}
+			}
+			go proxyBidirectional(conn, actualGrpcAddr)
+		}
+	}()
+
+	return func() {
+		close(stopped)
+		_ = listener.Close()
+	}
+}
+
+func proxyBidirectional(inbound net.Conn, targetAddr string) {
+	outbound, err := net.Dial("tcp", targetAddr)
+	if err != nil {
+		_ = inbound.Close()
+		return
+	}
+
+	go func() {
+		_, _ = io.Copy(outbound, inbound)
+		_ = outbound.Close()
+	}()
+	_, _ = io.Copy(inbound, outbound)
+	_ = inbound.Close()
+}

test/volume_server/grpc/data_rw_test.go

@@ -2,6 +2,7 @@ package volume_server_grpc_test
 
 import (
 	"context"
+	"net/http"
 	"strings"
 	"testing"
 	"time"
@@ -144,3 +145,66 @@ func TestReadNeedleBlobAndMetaInvalidOffsets(t *testing.T) {
 		t.Fatalf("ReadNeedleMeta should fail for invalid offset")
 	}
 }
+
+func TestReadNeedleMetaUnsupportedForEcMountedVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(93)
+	const needleID = uint64(880002)
+	const cookie = uint32(0x11AA22BB)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-meta-unsupported"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeUnmount failed: %v", err)
+	}
+
+	_, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
+		VolumeId: volumeID,
+		NeedleId: needleID,
+		Offset:   0,
+		Size:     128,
+	})
+	if err == nil {
+		t.Fatalf("ReadNeedleMeta should fail when only EC shards are mounted")
+	}
+	if !strings.Contains(strings.ToLower(err.Error()), "ec shards is not supported") {
+		t.Fatalf("ReadNeedleMeta EC-only error mismatch: %v", err)
+	}
+}

test/volume_server/grpc/data_stream_success_test.go

@@ -3,6 +3,7 @@ package volume_server_grpc_test
 import (
 	"context"
 	"io"
+	"net/http"
 	"strings"
 	"testing"
 	"time"
@@ -230,6 +231,71 @@ func TestReadAllNeedlesExistingThenMissingVolumeAbortsStream(t *testing.T) {
 	}
 }
 
+func TestReadAllNeedlesStreamsAcrossMultipleVolumes(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeIDA = uint32(86)
+	const volumeIDB = uint32(87)
+	const needleIDA = uint64(445561)
+	const needleIDB = uint64(445562)
+	framework.AllocateVolume(t, grpcClient, volumeIDA, "")
+	framework.AllocateVolume(t, grpcClient, volumeIDB, "")
+
+	client := framework.NewHTTPClient()
+	fidA := framework.NewFileID(volumeIDA, needleIDA, 0xAA11CC22)
+	fidB := framework.NewFileID(volumeIDB, needleIDB, 0xBB22DD33)
+	payloadA := "read-all-multi-volume-a"
+	payloadB := "read-all-multi-volume-b"
+
+	uploadA := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fidA, []byte(payloadA))
+	_ = framework.ReadAllAndClose(t, uploadA)
+	if uploadA.StatusCode != http.StatusCreated {
+		t.Fatalf("upload A expected 201, got %d", uploadA.StatusCode)
+	}
+	uploadB := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fidB, []byte(payloadB))
+	_ = framework.ReadAllAndClose(t, uploadB)
+	if uploadB.StatusCode != http.StatusCreated {
+		t.Fatalf("upload B expected 201, got %d", uploadB.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{
+		VolumeIds: []uint32{volumeIDA, volumeIDB},
+	})
+	if err != nil {
+		t.Fatalf("ReadAllNeedles start failed: %v", err)
+	}
+
+	seen := map[uint64]string{}
+	for {
+		msg, recvErr := stream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("ReadAllNeedles recv failed: %v", recvErr)
+		}
+		if msg.GetNeedleId() == needleIDA || msg.GetNeedleId() == needleIDB {
+			seen[msg.GetNeedleId()] = string(msg.GetNeedleBlob())
+		}
+	}
+
+	if got := seen[needleIDA]; got != payloadA {
+		t.Fatalf("ReadAllNeedles missing/mismatched payload for volume A needle: got %q want %q", got, payloadA)
+	}
+	if got := seen[needleIDB]; got != payloadB {
+		t.Fatalf("ReadAllNeedles missing/mismatched payload for volume B needle: got %q want %q", got, payloadB)
+	}
+}
+
 func copyFileBytes(t testing.TB, grpcClient volume_server_pb.VolumeServerClient, req *volume_server_pb.CopyFileRequest) []byte {
 	t.Helper()
 

test/volume_server/grpc/scrub_query_test.go

@@ -280,6 +280,54 @@ func TestQueryJsonSuccessAndCsvNoOutput(t *testing.T) {
 	}
 }
 
+func TestQueryCsvInputWithCsvPayloadStillReturnsNoRows(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(67)
+	const needleID = uint64(777004)
+	const cookie = uint32(0xDCDCADAD)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	csvPayload := []byte("name,score\nalice,12\nbob,3\n")
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, csvPayload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	csvStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
+		FromFileIds: []string{fid},
+		Selections:  []string{"score"},
+		Filter: &volume_server_pb.QueryRequest_Filter{
+			Field:   "score",
+			Operand: ">",
+			Value:   "10",
+		},
+		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
+			CsvInput: &volume_server_pb.QueryRequest_InputSerialization_CSVInput{},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Query csv start failed: %v", err)
+	}
+
+	_, err = csvStream.Recv()
+	if err != io.EOF {
+		t.Fatalf("Query csv with csv payload expected EOF with no rows, got: %v", err)
+	}
+}
+
 func TestQueryJsonNoMatchReturnsEmptyStripe(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test in short mode")

test/volume_server/grpc/tail_test.go

@@ -12,6 +12,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
 	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 func TestVolumeTailSenderMissingVolume(t *testing.T) {
@@ -91,6 +93,36 @@ func TestVolumeTailReceiverMissingVolume(t *testing.T) {
 	}
 }
 
+func TestVolumeTailReceiverSourceUnavailable(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(89)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
+		VolumeId:           volumeID,
+		SourceVolumeServer: "127.0.0.1:19999.29999",
+		SinceNs:            0,
+		IdleTimeoutSeconds: 1,
+	})
+	if err == nil {
+		t.Fatalf("VolumeTailReceiver should fail when source volume server is unavailable")
+	}
+	lowered := strings.ToLower(err.Error())
+	if !strings.Contains(lowered, "dial") && !strings.Contains(lowered, "unavailable") && !strings.Contains(lowered, "connection refused") {
+		t.Fatalf("VolumeTailReceiver source-unavailable error mismatch: %v", err)
+	}
+}
+
 func TestVolumeTailReceiverReplicatesSourceUpdates(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test in short mode")
@@ -204,3 +236,46 @@ func TestVolumeTailSenderLargeNeedleChunking(t *testing.T) {
 		t.Fatalf("VolumeTailSender expected a final data chunk marked IsLastChunk=true")
 	}
 }
+
+func TestVolumeTailSenderStreamCancellation(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(74)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithCancel(context.Background())
+	stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{
+		VolumeId:           volumeID,
+		SinceNs:            0,
+		IdleTimeoutSeconds: 30,
+	})
+	if err != nil {
+		cancel()
+		t.Fatalf("VolumeTailSender start failed: %v", err)
+	}
+
+	firstMsg, err := stream.Recv()
+	if err != nil {
+		cancel()
+		t.Fatalf("VolumeTailSender first recv failed: %v", err)
+	}
+	if !firstMsg.GetIsLastChunk() {
+		cancel()
+		t.Fatalf("expected heartbeat first message before cancellation")
+	}
+
+	cancel()
+	_, err = stream.Recv()
+	if err == nil {
+		t.Fatalf("VolumeTailSender recv after cancellation expected error")
+	}
+	if status.Code(err) != codes.Canceled {
+		t.Fatalf("VolumeTailSender cancellation expected grpc code Canceled, got %v (%v)", status.Code(err), err)
+	}
+}

test/volume_server/http/admin_test.go

@@ -164,6 +164,26 @@ func TestUiIndexNotExposedWhenJwtSigningEnabled(t *testing.T) {
 	}
 }
 
+func TestUiIndexExposedWhenJwtSigningEnabledAndAccessUITrue(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P3()
+	profile.AccessUI = true
+	cluster := framework.StartSingleVolumeCluster(t, profile)
+	client := framework.NewHTTPClient()
+
+	resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
+	body := framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("expected /ui/index.html to be exposed when access.ui=true under JWT profile, got %d body=%s", resp.StatusCode, string(body))
+	}
+	if !strings.Contains(strings.ToLower(string(body)), "volume") {
+		t.Fatalf("ui page does not look like volume status page")
+	}
+}
+
 func mustNewRequest(t testing.TB, method, url string) *http.Request {
 	t.Helper()
 	req, err := http.NewRequest(method, url, nil)

test/volume_server/http/public_cors_methods_test.go

@@ -251,6 +251,50 @@ func TestUnsupportedMethodConnectParity(t *testing.T) {
 	}
 }
 
+func TestUnsupportedMethodMkcolParity(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P2())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(88)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 124004, 0x06060606)
+	client := framework.NewHTTPClient()
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("mkcol-method-check"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	adminReq := mustNewRequest(t, "MKCOL", clusterHarness.VolumeAdminURL()+"/"+fid)
+	adminResp := framework.DoRequest(t, client, adminReq)
+	_ = framework.ReadAllAndClose(t, adminResp)
+	if adminResp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("admin MKCOL expected 400, got %d", adminResp.StatusCode)
+	}
+
+	publicReq := mustNewRequest(t, "MKCOL", clusterHarness.VolumePublicURL()+"/"+fid)
+	publicResp := framework.DoRequest(t, client, publicReq)
+	_ = framework.ReadAllAndClose(t, publicResp)
+	if publicResp.StatusCode != http.StatusOK {
+		t.Fatalf("public MKCOL expected passthrough 200, got %d", publicResp.StatusCode)
+	}
+
+	verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
+	verifyBody := framework.ReadAllAndClose(t, verifyResp)
+	if verifyResp.StatusCode != http.StatusOK {
+		t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode)
+	}
+	if string(verifyBody) != "mkcol-method-check" {
+		t.Fatalf("MKCOL should not mutate data, got %q", string(verifyBody))
+	}
+}
+
 func TestPublicPortHeadReadParity(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test in short mode")

test/volume_server/http/read_deleted_test.go

@@ -2,6 +2,7 @@ package volume_server_http_test
 
 import (
 	"net/http"
+	"strconv"
 	"testing"
 
 	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
@@ -51,4 +52,17 @@ func TestReadDeletedQueryReturnsDeletedNeedleData(t *testing.T) {
 	if string(readDeletedBody) != string(payload) {
 		t.Fatalf("readDeleted body mismatch: got %q want %q", string(readDeletedBody), string(payload))
 	}
+
+	headReadDeletedReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true")
+	headReadDeletedResp := framework.DoRequest(t, client, headReadDeletedReq)
+	headReadDeletedBody := framework.ReadAllAndClose(t, headReadDeletedResp)
+	if headReadDeletedResp.StatusCode != http.StatusOK {
+		t.Fatalf("HEAD with readDeleted=true expected 200, got %d", headReadDeletedResp.StatusCode)
+	}
+	if len(headReadDeletedBody) != 0 {
+		t.Fatalf("HEAD with readDeleted=true expected empty body, got %d bytes", len(headReadDeletedBody))
+	}
+	if got := headReadDeletedResp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) {
+		t.Fatalf("HEAD with readDeleted=true content-length mismatch: got %q want %d", got, len(payload))
+	}
 }

test/volume_server/http/read_path_variants_test.go

@@ -164,6 +164,10 @@ func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
 	if lastModified == "" {
 		t.Fatalf("baseline read expected Last-Modified header")
 	}
+	etag := baselineResp.Header.Get("ETag")
+	if etag == "" {
+		t.Fatalf("baseline read expected ETag header")
+	}
 
 	precedenceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
 	precedenceReq.Header.Set("If-Modified-Since", lastModified)
@@ -188,4 +192,28 @@ func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
 	if string(invalidIMSBody) != string(payload) {
 		t.Fatalf("invalid If-Modified-Since fallback body mismatch: got %q want %q", string(invalidIMSBody), string(payload))
 	}
+
+	headIMSPrecedenceReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fid)
+	headIMSPrecedenceReq.Header.Set("If-Modified-Since", lastModified)
+	headIMSPrecedenceReq.Header.Set("If-None-Match", "\"definitely-different-etag\"")
+	headIMSPrecedenceResp := framework.DoRequest(t, client, headIMSPrecedenceReq)
+	headIMSPrecedenceBody := framework.ReadAllAndClose(t, headIMSPrecedenceResp)
+	if headIMSPrecedenceResp.StatusCode != http.StatusNotModified {
+		t.Fatalf("HEAD conditional precedence expected 304 when If-Modified-Since matches, got %d", headIMSPrecedenceResp.StatusCode)
+	}
+	if len(headIMSPrecedenceBody) != 0 {
+		t.Fatalf("HEAD conditional precedence expected empty body, got %d bytes", len(headIMSPrecedenceBody))
+	}
+
+	headETagPrecedenceReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fid)
+	headETagPrecedenceReq.Header.Set("If-Modified-Since", "Thu, 01 Jan 1970 00:00:00 GMT")
+	headETagPrecedenceReq.Header.Set("If-None-Match", etag)
+	headETagPrecedenceResp := framework.DoRequest(t, client, headETagPrecedenceReq)
+	headETagPrecedenceBody := framework.ReadAllAndClose(t, headETagPrecedenceResp)
+	if headETagPrecedenceResp.StatusCode != http.StatusNotModified {
+		t.Fatalf("HEAD conditional precedence expected 304 when If-None-Match matches, got %d", headETagPrecedenceResp.StatusCode)
+	}
+	if len(headETagPrecedenceBody) != 0 {
+		t.Fatalf("HEAD conditional etag precedence expected empty body, got %d bytes", len(headETagPrecedenceBody))
+	}
 }

test/volume_server/http/write_error_variants_test.go

@@ -1,6 +1,7 @@
 package volume_server_http_test
 
 import (
+	"bytes"
 	"net/http"
 	"strings"
 	"testing"
@@ -72,3 +73,64 @@ func TestWriteMalformedMultipartAndMD5Mismatch(t *testing.T) {
 		t.Fatalf("content-md5 mismatch response should mention Content-MD5, got %q", string(md5MismatchBody))
 	}
 }
+
+func TestWriteRejectsPayloadOverFileSizeLimit(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	profile := matrix.P1()
+	profile.FileSizeLimitMB = 1
+	clusterHarness := framework.StartSingleVolumeCluster(t, profile)
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(99)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 772002, 0x2A3B4C5D)
+	oversizedPayload := bytes.Repeat([]byte("z"), 1024*1024+1)
+
+	oversizedReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, oversizedPayload)
+	oversizedResp := framework.DoRequest(t, client, oversizedReq)
+	oversizedBody := framework.ReadAllAndClose(t, oversizedResp)
+	if oversizedResp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("oversized write expected 400, got %d", oversizedResp.StatusCode)
+	}
+	if !strings.Contains(strings.ToLower(string(oversizedBody)), "limited") {
+		t.Fatalf("oversized write response should mention limit, got %q", string(oversizedBody))
+	}
+}
+
+func TestReplicatedWriteFailsWhenReplicaRequirementsNotMet(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(109)
+	framework.AllocateVolumeWithReplication(t, grpcClient, volumeID, "", "001")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 772003, 0x3A4B5C6D)
+	payload := []byte("replicated-write-failure-path")
+
+	writeResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
+	writeBody := framework.ReadAllAndClose(t, writeResp)
+	if writeResp.StatusCode != http.StatusInternalServerError {
+		t.Fatalf("replicated write with unmet replication requirements expected 500, got %d body=%s", writeResp.StatusCode, string(writeBody))
+	}
+	if !strings.Contains(strings.ToLower(string(writeBody)), "replica") {
+		t.Fatalf("replicated write failure response should mention replica write failure, got %q", string(writeBody))
+	}
+
+	readResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
+	_ = framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusNotFound {
+		t.Fatalf("local read after failed replicate write expected 404 (not committed locally), got %d", readResp.StatusCode)
+	}
+}

test/volume_server/matrix/config_profiles.go

@@ -12,12 +12,14 @@ type Profile struct {
 	EnableJWT      bool
 	JWTSigningKey  string
 	JWTReadKey     string
+	AccessUI       bool
 	EnableMaintain bool
 
 	ConcurrentUploadLimitMB   int
 	ConcurrentDownloadLimitMB int
 	InflightUploadTimeout     time.Duration
 	InflightDownloadTimeout   time.Duration
+	FileSizeLimitMB           int
 
 	ReplicatedLayout bool
 	HasErasureCoding bool

unmaintained/disk/disk_status.go

@@ -1,4 +1,5 @@
 //go:build !windows && !openbsd && !netbsd && !plan9 && !solaris
+// +build !windows,!openbsd,!netbsd,!plan9,!solaris
 
 package main
 

unmaintained/stress/ec_stress_runner.go

@@ -1,515 +0,0 @@
-package main
-
-import (
-	"context"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	mrand "math/rand"
-	"net/http"
-	"net/url"
-	"os/signal"
-	"path"
-	"sort"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
-)
-
-type config struct {
-	MasterAddresses    []string
-	FilerURL           string
-	PathPrefix         string
-	Collection         string
-	FileSizeBytes      int64
-	BatchSize          int
-	WriteInterval      time.Duration
-	CleanupInterval    time.Duration
-	EcMinAge           time.Duration
-	MaxCleanupPerCycle int
-	RequestTimeout     time.Duration
-	MaxRuntime         time.Duration
-	DryRun             bool
-}
-
-type runner struct {
-	cfg config
-
-	httpClient     *http.Client
-	grpcDialOption grpc.DialOption
-
-	mu            sync.Mutex
-	sequence      int64
-	ecFirstSeenAt map[uint32]time.Time
-	rng           *mrand.Rand
-}
-
-type ecVolumeInfo struct {
-	Collection string
-	NodeShards map[pb.ServerAddress][]uint32
-}
-
-type ecCleanupCandidate struct {
-	VolumeID    uint32
-	FirstSeenAt time.Time
-	Info        *ecVolumeInfo
-}
-
-func main() {
-	cfg, err := loadConfig()
-	if err != nil {
-		log.Fatalf("invalid flags: %v", err)
-	}
-
-	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
-	defer stop()
-
-	if cfg.MaxRuntime > 0 {
-		runCtx, cancel := context.WithTimeout(ctx, cfg.MaxRuntime)
-		defer cancel()
-		ctx = runCtx
-	}
-
-	r := &runner{
-		cfg:            cfg,
-		httpClient:     &http.Client{Timeout: cfg.RequestTimeout},
-		grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
-		ecFirstSeenAt:  make(map[uint32]time.Time),
-		rng:            mrand.New(mrand.NewSource(time.Now().UnixNano())),
-	}
-
-	log.Printf(
-		"starting EC stress runner: masters=%s filer=%s prefix=%s collection=%s file_size=%d batch=%d write_interval=%s cleanup_interval=%s ec_min_age=%s max_cleanup=%d dry_run=%v",
-		strings.Join(cfg.MasterAddresses, ","),
-		cfg.FilerURL,
-		cfg.PathPrefix,
-		cfg.Collection,
-		cfg.FileSizeBytes,
-		cfg.BatchSize,
-		cfg.WriteInterval,
-		cfg.CleanupInterval,
-		cfg.EcMinAge,
-		cfg.MaxCleanupPerCycle,
-		cfg.DryRun,
-	)
-
-	if err := r.run(ctx); err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
-		log.Fatalf("runner stopped with error: %v", err)
-	}
-
-	log.Printf("runner stopped")
-}
-
-func loadConfig() (config, error) {
-	var masters string
-	cfg := config{}
-
-	flag.StringVar(&masters, "masters", "127.0.0.1:9333", "comma-separated master server addresses")
-	flag.StringVar(&cfg.FilerURL, "filer", "http://127.0.0.1:8888", "filer base URL")
-	flag.StringVar(&cfg.PathPrefix, "path_prefix", "/tmp/ec-stress", "filer path prefix for generated files")
-	flag.StringVar(&cfg.Collection, "collection", "ec_stress", "target collection for stress data")
-
-	fileSizeMB := flag.Int("file_size_mb", 8, "size per generated file in MB")
-	flag.IntVar(&cfg.BatchSize, "batch_size", 4, "files generated per write cycle")
-	flag.DurationVar(&cfg.WriteInterval, "write_interval", 5*time.Second, "interval between write cycles")
-	flag.DurationVar(&cfg.CleanupInterval, "cleanup_interval", 2*time.Minute, "interval between EC cleanup cycles")
-	flag.DurationVar(&cfg.EcMinAge, "ec_min_age", 30*time.Minute, "minimum observed EC age before deletion")
-	flag.IntVar(&cfg.MaxCleanupPerCycle, "max_cleanup_per_cycle", 4, "maximum EC volumes deleted per cleanup cycle")
-	flag.DurationVar(&cfg.RequestTimeout, "request_timeout", 20*time.Second, "HTTP/gRPC request timeout")
-	flag.DurationVar(&cfg.MaxRuntime, "max_runtime", 0, "maximum run duration; 0 means run until interrupted")
-	flag.BoolVar(&cfg.DryRun, "dry_run", false, "log actions without deleting EC shards")
-	flag.Parse()
-
-	cfg.MasterAddresses = splitNonEmpty(masters)
-	cfg.FileSizeBytes = int64(*fileSizeMB) * 1024 * 1024
-
-	if len(cfg.MasterAddresses) == 0 {
-		return cfg, fmt.Errorf("at least one master is required")
-	}
-	if cfg.FileSizeBytes <= 0 {
-		return cfg, fmt.Errorf("file_size_mb must be positive")
-	}
-	if cfg.BatchSize <= 0 {
-		return cfg, fmt.Errorf("batch_size must be positive")
-	}
-	if cfg.WriteInterval <= 0 {
-		return cfg, fmt.Errorf("write_interval must be positive")
-	}
-	if cfg.CleanupInterval <= 0 {
-		return cfg, fmt.Errorf("cleanup_interval must be positive")
-	}
-	if cfg.EcMinAge < 0 {
-		return cfg, fmt.Errorf("ec_min_age must be zero or positive")
-	}
-	// Note: EcMinAge == 0 intentionally disables the age guard, making EC volumes eligible for cleanup immediately.
-	if cfg.MaxCleanupPerCycle <= 0 {
-		return cfg, fmt.Errorf("max_cleanup_per_cycle must be positive")
-	}
-	if cfg.RequestTimeout <= 0 {
-		return cfg, fmt.Errorf("request_timeout must be positive")
-	}
-
-	cfg.PathPrefix = ensureLeadingSlash(strings.TrimSpace(cfg.PathPrefix))
-	cfg.Collection = strings.TrimSpace(cfg.Collection)
-
-	cfg.FilerURL = strings.TrimRight(strings.TrimSpace(cfg.FilerURL), "/")
-	if cfg.FilerURL == "" {
-		return cfg, fmt.Errorf("filer URL is required")
-	}
-
-	if _, err := url.ParseRequestURI(cfg.FilerURL); err != nil {
-		return cfg, fmt.Errorf("invalid filer URL %q: %w", cfg.FilerURL, err)
-	}
-
-	return cfg, nil
-}
-
-func (r *runner) run(ctx context.Context) error {
-	writeTicker := time.NewTicker(r.cfg.WriteInterval)
-	defer writeTicker.Stop()
-
-	cleanupTicker := time.NewTicker(r.cfg.CleanupInterval)
-	defer cleanupTicker.Stop()
-
-	r.runWriteCycle(ctx)
-	r.runCleanupCycle(ctx)
-
-	for {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-writeTicker.C:
-			r.runWriteCycle(ctx)
-		case <-cleanupTicker.C:
-			r.runCleanupCycle(ctx)
-		}
-	}
-}
-
-func (r *runner) runWriteCycle(ctx context.Context) {
-	for i := 0; i < r.cfg.BatchSize; i++ {
-		if ctx.Err() != nil {
-			return
-		}
-		if err := r.uploadOneFile(ctx); err != nil {
-			log.Printf("upload failed: %v", err)
-		}
-	}
-}
-
-func (r *runner) uploadOneFile(ctx context.Context) error {
-	sequence := r.nextSequence()
-	filePath := path.Join(r.cfg.PathPrefix, fmt.Sprintf("ec-stress-%d-%d.bin", time.Now().UnixNano(), sequence))
-	fileURL := r.cfg.FilerURL + filePath
-	if r.cfg.Collection != "" {
-		fileURL += "?collection=" + url.QueryEscape(r.cfg.Collection)
-	}
-
-	uploadCtx, cancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
-	defer cancel()
-
-	body := io.LimitReader(r.rng, r.cfg.FileSizeBytes)
-	request, err := http.NewRequestWithContext(uploadCtx, http.MethodPut, fileURL, body)
-	if err != nil {
-		return err
-	}
-	request.ContentLength = r.cfg.FileSizeBytes
-	request.Header.Set("Content-Type", "application/octet-stream")
-
-	response, err := r.httpClient.Do(request)
-	if err != nil {
-		return err
-	}
-	defer response.Body.Close()
-
-	io.Copy(io.Discard, response.Body)
-	if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
-		return fmt.Errorf("upload %s returned %s", filePath, response.Status)
-	}
-
-	log.Printf("uploaded %s size=%d", filePath, r.cfg.FileSizeBytes)
-	return nil
-}
-
-func (r *runner) runCleanupCycle(ctx context.Context) {
-	volumeList, err := r.fetchVolumeList(ctx)
-	if err != nil {
-		log.Printf("cleanup skipped: fetch volume list failed: %v", err)
-		return
-	}
-	if volumeList == nil || volumeList.TopologyInfo == nil {
-		log.Printf("cleanup skipped: topology is empty")
-		return
-	}
-
-	ecVolumes := collectEcVolumes(volumeList.TopologyInfo, r.cfg.Collection)
-	candidates := r.selectCleanupCandidates(ecVolumes)
-	if len(candidates) == 0 {
-		log.Printf("cleanup: no EC volume candidate aged >= %s in collection=%q", r.cfg.EcMinAge, r.cfg.Collection)
-		return
-	}
-
-	log.Printf("cleanup: deleting up to %d EC volumes (found=%d)", r.cfg.MaxCleanupPerCycle, len(candidates))
-	deleted := 0
-	for _, candidate := range candidates {
-		if ctx.Err() != nil {
-			return
-		}
-
-		if r.cfg.DryRun {
-			log.Printf(
-				"cleanup dry-run: would delete EC volume=%d collection=%q seen_for=%s nodes=%d",
-				candidate.VolumeID,
-				candidate.Info.Collection,
-				time.Since(candidate.FirstSeenAt).Round(time.Second),
-				len(candidate.Info.NodeShards),
-			)
-			continue
-		}
-
-		if err := r.deleteEcVolume(ctx, candidate.VolumeID, candidate.Info); err != nil {
-			log.Printf("cleanup volume=%d failed: %v", candidate.VolumeID, err)
-			continue
-		}
-
-		deleted++
-		r.mu.Lock()
-		delete(r.ecFirstSeenAt, candidate.VolumeID)
-		r.mu.Unlock()
-		log.Printf("cleanup volume=%d completed", candidate.VolumeID)
-	}
-
-	log.Printf("cleanup finished: deleted=%d attempted=%d", deleted, len(candidates))
-}
-
-func (r *runner) fetchVolumeList(ctx context.Context) (*master_pb.VolumeListResponse, error) {
-	var lastErr error
-	for _, master := range r.cfg.MasterAddresses {
-		masterAddress := strings.TrimSpace(master)
-		if masterAddress == "" {
-			continue
-		}
-
-		var response *master_pb.VolumeListResponse
-		err := pb.WithMasterClient(false, pb.ServerAddress(masterAddress), r.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
-			callCtx, cancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
-			defer cancel()
-
-			resp, callErr := client.VolumeList(callCtx, &master_pb.VolumeListRequest{})
-			if callErr != nil {
-				return callErr
-			}
-			response = resp
-			return nil
-		})
-		if err == nil {
-			return response, nil
-		}
-
-		lastErr = err
-	}
-
-	if lastErr == nil {
-		lastErr = fmt.Errorf("no valid master address")
-	}
-	return nil, lastErr
-}
-
-func collectEcVolumes(topology *master_pb.TopologyInfo, collection string) map[uint32]*ecVolumeInfo {
-	normalizedCollection := strings.TrimSpace(collection)
-	volumeShardSets := make(map[uint32]map[pb.ServerAddress]map[uint32]struct{})
-	volumeCollection := make(map[uint32]string)
-
-	for _, dc := range topology.GetDataCenterInfos() {
-		for _, rack := range dc.GetRackInfos() {
-			for _, node := range rack.GetDataNodeInfos() {
-				server := pb.NewServerAddressFromDataNode(node)
-				for _, disk := range node.GetDiskInfos() {
-					for _, shardInfo := range disk.GetEcShardInfos() {
-						if shardInfo == nil || shardInfo.Id == 0 {
-							continue
-						}
-						if normalizedCollection != "" && strings.TrimSpace(shardInfo.Collection) != normalizedCollection {
-							continue
-						}
-
-						shards := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo).IdsUint32()
-						if len(shards) == 0 {
-							continue
-						}
-
-						perVolume := volumeShardSets[shardInfo.Id]
-						if perVolume == nil {
-							perVolume = make(map[pb.ServerAddress]map[uint32]struct{})
-							volumeShardSets[shardInfo.Id] = perVolume
-						}
-						perNode := perVolume[server]
-						if perNode == nil {
-							perNode = make(map[uint32]struct{})
-							perVolume[server] = perNode
-						}
-						for _, shardID := range shards {
-							perNode[shardID] = struct{}{}
-						}
-						volumeCollection[shardInfo.Id] = shardInfo.Collection
-					}
-				}
-			}
-		}
-	}
-
-	result := make(map[uint32]*ecVolumeInfo, len(volumeShardSets))
-	for volumeID, perNode := range volumeShardSets {
-		info := &ecVolumeInfo{
-			Collection: volumeCollection[volumeID],
-			NodeShards: make(map[pb.ServerAddress][]uint32, len(perNode)),
-		}
-		for server, shardSet := range perNode {
-			shardIDs := make([]uint32, 0, len(shardSet))
-			for shardID := range shardSet {
-				shardIDs = append(shardIDs, shardID)
-			}
-			sort.Slice(shardIDs, func(i, j int) bool { return shardIDs[i] < shardIDs[j] })
-			info.NodeShards[server] = shardIDs
-		}
-		result[volumeID] = info
-	}
-
-	return result
-}
-
-func (r *runner) selectCleanupCandidates(ecVolumes map[uint32]*ecVolumeInfo) []ecCleanupCandidate {
-	now := time.Now()
-
-	r.mu.Lock()
-	defer r.mu.Unlock()
-
-	for volumeID := range ecVolumes {
-		if _, exists := r.ecFirstSeenAt[volumeID]; !exists {
-			r.ecFirstSeenAt[volumeID] = now
-		}
-	}
-	for volumeID := range r.ecFirstSeenAt {
-		if _, exists := ecVolumes[volumeID]; !exists {
-			delete(r.ecFirstSeenAt, volumeID)
-		}
-	}
-
-	candidates := make([]ecCleanupCandidate, 0, len(ecVolumes))
-	for volumeID, info := range ecVolumes {
-		firstSeenAt := r.ecFirstSeenAt[volumeID]
-		if r.cfg.EcMinAge > 0 && now.Sub(firstSeenAt) < r.cfg.EcMinAge {
-			continue
-		}
-		candidates = append(candidates, ecCleanupCandidate{
-			VolumeID:    volumeID,
-			FirstSeenAt: firstSeenAt,
-			Info:        info,
-		})
-	}
-
-	sort.Slice(candidates, func(i, j int) bool {
-		if candidates[i].FirstSeenAt.Equal(candidates[j].FirstSeenAt) {
-			return candidates[i].VolumeID < candidates[j].VolumeID
-		}
-		return candidates[i].FirstSeenAt.Before(candidates[j].FirstSeenAt)
-	})
-
-	if len(candidates) > r.cfg.MaxCleanupPerCycle {
-		candidates = candidates[:r.cfg.MaxCleanupPerCycle]
-	}
-	return candidates
-}
-
-func (r *runner) deleteEcVolume(ctx context.Context, volumeID uint32, info *ecVolumeInfo) error {
-	if info == nil {
-		return fmt.Errorf("ec volume %d has no topology info", volumeID)
-	}
-
-	failureCount := 0
-	for server, shardIDs := range info.NodeShards {
-		err := pb.WithVolumeServerClient(false, server, r.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
-			unmountCtx, unmountCancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
-			defer unmountCancel()
-			if _, err := client.VolumeEcShardsUnmount(unmountCtx, &volume_server_pb.VolumeEcShardsUnmountRequest{
-				VolumeId: volumeID,
-				ShardIds: shardIDs,
-			}); err != nil {
-				log.Printf("volume %d ec shards unmount on %s failed: %v", volumeID, server, err)
-			}
-
-			if len(shardIDs) > 0 {
-				deleteCtx, deleteCancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
-				defer deleteCancel()
-				if _, err := client.VolumeEcShardsDelete(deleteCtx, &volume_server_pb.VolumeEcShardsDeleteRequest{
-					VolumeId:   volumeID,
-					Collection: r.cfg.Collection,
-					ShardIds:   shardIDs,
-				}); err != nil {
-					return err
-				}
-			}
-
-			finalDeleteCtx, finalDeleteCancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
-			defer finalDeleteCancel()
-			if _, err := client.VolumeDelete(finalDeleteCtx, &volume_server_pb.VolumeDeleteRequest{
-				VolumeId: volumeID,
-			}); err != nil {
-				log.Printf("volume %d delete on %s failed: %v", volumeID, server, err)
-			}
-			return nil
-		})
-
-		if err != nil {
-			failureCount++
-			log.Printf("cleanup volume=%d server=%s shards=%v failed: %v", volumeID, server, shardIDs, err)
-		}
-	}
-
-	if failureCount == len(info.NodeShards) && failureCount > 0 {
-		return fmt.Errorf("all shard deletions failed for volume %d", volumeID)
-	}
-	if failureCount > 0 {
-		return fmt.Errorf("partial shard deletion failure for volume %d", volumeID)
-	}
-	return nil
-}
-
-func (r *runner) nextSequence() int64 {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	r.sequence++
-	return r.sequence
-}
-
-func splitNonEmpty(value string) []string {
-	parts := strings.Split(value, ",")
-	result := make([]string, 0, len(parts))
-	for _, part := range parts {
-		trimmed := strings.TrimSpace(part)
-		if trimmed != "" {
-			result = append(result, trimmed)
-		}
-	}
-	return result
-}
-
-func ensureLeadingSlash(value string) string {
-	if value == "" {
-		return "/"
-	}
-	if strings.HasPrefix(value, "/") {
-		return value
-	}
-	return "/" + value
-}

weed/admin/Makefile

@@ -23,7 +23,7 @@ install-templ:
 .PHONY: generate
 generate: install-templ
 	@echo "Generating templ files..."
-	@templ generate ./view
+	@cd $(BUILD_DIR) && templ generate
 	@echo "Generated: $(TEMPL_GO_FILES)"
 
 # Clean generated files

weed/admin/dash/admin_data.go

@@ -10,7 +10,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/iam"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 )
 
@@ -234,7 +233,7 @@ func (s *AdminServer) getMasterNodesStatus() []MasterNode {
 	currentMaster := s.masterClient.GetMaster(context.Background())
 	if currentMaster != "" {
 		masterNodes = append(masterNodes, MasterNode{
-			Address:  pb.ServerAddress(currentMaster).ToHttpAddress(),
+			Address:  string(currentMaster),
 			IsLeader: isLeader,
 		})
 	}

weed/admin/dash/plugin_api.go

@@ -1,735 +0,0 @@
-package dash
-
-import (
-	"context"
-	"crypto/rand"
-	"encoding/hex"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"sort"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/gin-gonic/gin"
-	"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-	"google.golang.org/protobuf/encoding/protojson"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/timestamppb"
-)
-
-const (
-	defaultPluginDetectionTimeout = 45 * time.Second
-	defaultPluginExecutionTimeout = 90 * time.Second
-	maxPluginDetectionTimeout     = 5 * time.Minute
-	maxPluginExecutionTimeout     = 10 * time.Minute
-	defaultPluginRunTimeout       = 5 * time.Minute
-	maxPluginRunTimeout           = 30 * time.Minute
-)
-
-// GetPluginStatusAPI returns plugin status.
-func (s *AdminServer) GetPluginStatusAPI(c *gin.Context) {
-	plugin := s.GetPlugin()
-	if plugin == nil {
-		c.JSON(http.StatusOK, gin.H{
-			"enabled":          false,
-			"worker_grpc_port": s.GetWorkerGrpcPort(),
-		})
-		return
-	}
-
-	c.JSON(http.StatusOK, gin.H{
-		"enabled":          true,
-		"configured":       plugin.IsConfigured(),
-		"base_dir":         plugin.BaseDir(),
-		"worker_count":     len(plugin.ListWorkers()),
-		"worker_grpc_port": s.GetWorkerGrpcPort(),
-	})
-}
-
-// GetPluginWorkersAPI returns currently connected plugin workers.
-func (s *AdminServer) GetPluginWorkersAPI(c *gin.Context) {
-	workers := s.GetPluginWorkers()
-	if workers == nil {
-		c.JSON(http.StatusOK, []interface{}{})
-		return
-	}
-	c.JSON(http.StatusOK, workers)
-}
-
-// GetPluginJobTypesAPI returns known plugin job types from workers and persisted data.
-func (s *AdminServer) GetPluginJobTypesAPI(c *gin.Context) {
-	jobTypes, err := s.ListPluginJobTypes()
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
-		return
-	}
-	if jobTypes == nil {
-		c.JSON(http.StatusOK, []interface{}{})
-		return
-	}
-	c.JSON(http.StatusOK, jobTypes)
-}
-
-// GetPluginJobsAPI returns tracked jobs for monitoring.
-func (s *AdminServer) GetPluginJobsAPI(c *gin.Context) {
-	jobType := strings.TrimSpace(c.Query("job_type"))
-	state := strings.TrimSpace(c.Query("state"))
-	limit := parsePositiveInt(c.Query("limit"), 200)
-	jobs := s.ListPluginJobs(jobType, state, limit)
-	if jobs == nil {
-		c.JSON(http.StatusOK, []interface{}{})
-		return
-	}
-	c.JSON(http.StatusOK, jobs)
-}
-
-// GetPluginJobAPI returns one tracked job.
-func (s *AdminServer) GetPluginJobAPI(c *gin.Context) {
-	jobID := strings.TrimSpace(c.Param("jobId"))
-	if jobID == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "jobId is required"})
-		return
-	}
-
-	job, found := s.GetPluginJob(jobID)
-	if !found {
-		c.JSON(http.StatusNotFound, gin.H{"error": "job not found"})
-		return
-	}
-	c.JSON(http.StatusOK, job)
-}
-
-// GetPluginJobDetailAPI returns detailed information for one tracked plugin job.
-func (s *AdminServer) GetPluginJobDetailAPI(c *gin.Context) {
-	jobID := strings.TrimSpace(c.Param("jobId"))
-	if jobID == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "jobId is required"})
-		return
-	}
-
-	activityLimit := parsePositiveInt(c.Query("activity_limit"), 500)
-	relatedLimit := parsePositiveInt(c.Query("related_limit"), 20)
-
-	detail, found, err := s.GetPluginJobDetail(jobID, activityLimit, relatedLimit)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
-		return
-	}
-	if !found || detail == nil {
-		c.JSON(http.StatusNotFound, gin.H{"error": "job detail not found"})
-		return
-	}
-
-	c.JSON(http.StatusOK, detail)
-}
-
-// GetPluginActivitiesAPI returns recent plugin activities.
-func (s *AdminServer) GetPluginActivitiesAPI(c *gin.Context) {
-	jobType := strings.TrimSpace(c.Query("job_type"))
-	limit := parsePositiveInt(c.Query("limit"), 500)
-	activities := s.ListPluginActivities(jobType, limit)
-	if activities == nil {
-		c.JSON(http.StatusOK, []interface{}{})
-		return
-	}
-	c.JSON(http.StatusOK, activities)
-}
-
-// GetPluginSchedulerStatesAPI returns per-job-type scheduler status for monitoring.
-func (s *AdminServer) GetPluginSchedulerStatesAPI(c *gin.Context) {
-	jobTypeFilter := strings.TrimSpace(c.Query("job_type"))
-
-	states, err := s.ListPluginSchedulerStates()
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
-		return
-	}
-
-	if jobTypeFilter != "" {
-		filtered := make([]interface{}, 0, len(states))
-		for _, state := range states {
-			if state.JobType == jobTypeFilter {
-				filtered = append(filtered, state)
-			}
-		}
-		c.JSON(http.StatusOK, filtered)
-		return
-	}
-
-	if states == nil {
-		c.JSON(http.StatusOK, []interface{}{})
-		return
-	}
-
-	c.JSON(http.StatusOK, states)
-}
-
-// RequestPluginJobTypeSchemaAPI asks a worker for one job type schema.
-func (s *AdminServer) RequestPluginJobTypeSchemaAPI(c *gin.Context) {
-	jobType := strings.TrimSpace(c.Param("jobType"))
-	if jobType == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"})
-		return
-	}
-
-	forceRefresh := c.DefaultQuery("force_refresh", "false") == "true"
-
-	ctx, cancel := context.WithTimeout(c.Request.Context(), defaultPluginDetectionTimeout)
-	defer cancel()
-	descriptor, err := s.RequestPluginJobTypeDescriptor(ctx, jobType, forceRefresh)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
-		return
-	}
-
-	renderProtoJSON(c, http.StatusOK, descriptor)
-}
-
-// GetPluginJobTypeDescriptorAPI returns persisted descriptor for a job type.
-func (s *AdminServer) GetPluginJobTypeDescriptorAPI(c *gin.Context) {
-	jobType := strings.TrimSpace(c.Param("jobType"))
-	if jobType == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"})
-		return
-	}
-
-	descriptor, err := s.LoadPluginJobTypeDescriptor(jobType)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
-		return
-	}
-	if descriptor == nil {
-		c.JSON(http.StatusNotFound, gin.H{"error": "descriptor not found"})
-		return
-	}
-
-	renderProtoJSON(c, http.StatusOK, descriptor)
-}
-
-// GetPluginJobTypeConfigAPI loads persisted config for a job type.
-func (s *AdminServer) GetPluginJobTypeConfigAPI(c *gin.Context) {
-	jobType := strings.TrimSpace(c.Param("jobType"))
-	if jobType == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"})
-		return
-	}
-
-	config, err := s.LoadPluginJobTypeConfig(jobType)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
-		return
-	}
-	if config == nil {
-		config = &plugin_pb.PersistedJobTypeConfig{
-			JobType:            jobType,
-			AdminConfigValues:  map[string]*plugin_pb.ConfigValue{},
-			WorkerConfigValues: map[string]*plugin_pb.ConfigValue{},
-			AdminRuntime:       &plugin_pb.AdminRuntimeConfig{},
-		}
-	}
-
-	renderProtoJSON(c, http.StatusOK, config)
-}
-
-// UpdatePluginJobTypeConfigAPI stores persisted config for a job type.
-func (s *AdminServer) UpdatePluginJobTypeConfigAPI(c *gin.Context) {
-	jobType := strings.TrimSpace(c.Param("jobType"))
-	if jobType == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"})
-		return
-	}
-
-	config := &plugin_pb.PersistedJobTypeConfig{}
-	if err := parseProtoJSONBody(c, config); err != nil {
-		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
-		return
-	}
-
-	config.JobType = jobType
-	if config.UpdatedAt == nil {
-		config.UpdatedAt = timestamppb.Now()
-	}
-	if config.AdminRuntime == nil {
-		config.AdminRuntime = &plugin_pb.AdminRuntimeConfig{}
-	}
-	if config.AdminConfigValues == nil {
-		config.AdminConfigValues = map[string]*plugin_pb.ConfigValue{}
-	}
-	if config.WorkerConfigValues == nil {
-		config.WorkerConfigValues = map[string]*plugin_pb.ConfigValue{}
-	}
-
-	username := c.GetString("username")
-	if username == "" {
-		username = "admin"
-	}
-	config.UpdatedBy = username
-
-	if err := s.SavePluginJobTypeConfig(config); err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
-		return
-	}
-
-	renderProtoJSON(c, http.StatusOK, config)
-}
-
-// GetPluginRunHistoryAPI returns bounded run history for a job type.
-func (s *AdminServer) GetPluginRunHistoryAPI(c *gin.Context) {
-	jobType := strings.TrimSpace(c.Param("jobType"))
-	if jobType == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"})
-		return
-	}
-
-	history, err := s.GetPluginRunHistory(jobType)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
-		return
-	}
-	if history == nil {
-		c.JSON(http.StatusOK, gin.H{
-			"job_type":          jobType,
-			"successful_runs":   []interface{}{},
-			"error_runs":        []interface{}{},
-			"last_updated_time": nil,
-		})
-		return
-	}
-
-	c.JSON(http.StatusOK, history)
-}
-
-// TriggerPluginDetectionAPI runs one detector for this job type and returns proposals.
-func (s *AdminServer) TriggerPluginDetectionAPI(c *gin.Context) {
-	jobType := strings.TrimSpace(c.Param("jobType"))
-	if jobType == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"})
-		return
-	}
-
-	var req struct {
-		ClusterContext json.RawMessage `json:"cluster_context"`
-		MaxResults     int32           `json:"max_results"`
-		TimeoutSeconds int             `json:"timeout_seconds"`
-	}
-
-	if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request body: " + err.Error()})
-		return
-	}
-
-	clusterContext, err := s.parseOrBuildClusterContext(req.ClusterContext)
-	if err != nil {
-		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
-		return
-	}
-
-	timeout := normalizeTimeout(req.TimeoutSeconds, defaultPluginDetectionTimeout, maxPluginDetectionTimeout)
-	ctx, cancel := context.WithTimeout(c.Request.Context(), timeout)
-	defer cancel()
-
-	report, err := s.RunPluginDetectionWithReport(ctx, jobType, clusterContext, req.MaxResults)
-	proposals := make([]*plugin_pb.JobProposal, 0)
-	requestID := ""
-	detectorWorkerID := ""
-	totalProposals := int32(0)
-	if report != nil {
-		proposals = report.Proposals
-		requestID = report.RequestID
-		detectorWorkerID = report.WorkerID
-		if report.Complete != nil {
-			totalProposals = report.Complete.TotalProposals
-		}
-	}
-
-	proposalPayloads := make([]map[string]interface{}, 0, len(proposals))
-	for _, proposal := range proposals {
-		payload, marshalErr := protoMessageToMap(proposal)
-		if marshalErr != nil {
-			glog.Warningf("failed to marshal proposal for jobType=%s: %v", jobType, marshalErr)
-			continue
-		}
-		proposalPayloads = append(proposalPayloads, payload)
-	}
-
-	sort.Slice(proposalPayloads, func(i, j int) bool {
-		iPriorityStr, _ := proposalPayloads[i]["priority"].(string)
-		jPriorityStr, _ := proposalPayloads[j]["priority"].(string)
-
-		iPriority := plugin_pb.JobPriority_value[iPriorityStr]
-		jPriority := plugin_pb.JobPriority_value[jPriorityStr]
-
-		if iPriority != jPriority {
-			return iPriority > jPriority
-		}
-		iID, _ := proposalPayloads[i]["proposal_id"].(string)
-		jID, _ := proposalPayloads[j]["proposal_id"].(string)
-		return iID < jID
-	})
-
-	activities := s.ListPluginActivities(jobType, 500)
-	filteredActivities := make([]interface{}, 0, len(activities))
-	if requestID != "" {
-		for i := len(activities) - 1; i >= 0; i-- {
-			activity := activities[i]
-			if activity.RequestID != requestID {
-				continue
-			}
-			filteredActivities = append(filteredActivities, activity)
-		}
-	}
-
-	response := gin.H{
-		"job_type":           jobType,
-		"request_id":         requestID,
-		"detector_worker_id": detectorWorkerID,
-		"total_proposals":    totalProposals,
-		"count":              len(proposalPayloads),
-		"proposals":          proposalPayloads,
-		"activities":         filteredActivities,
-	}
-
-	if err != nil {
-		response["error"] = err.Error()
-		c.JSON(http.StatusInternalServerError, response)
-		return
-	}
-
-	c.JSON(http.StatusOK, response)
-}
-
-// RunPluginJobTypeAPI runs full workflow for one job type: detect then dispatch detected jobs.
-func (s *AdminServer) RunPluginJobTypeAPI(c *gin.Context) {
-	jobType := strings.TrimSpace(c.Param("jobType"))
-	if jobType == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"})
-		return
-	}
-
-	var req struct {
-		ClusterContext json.RawMessage `json:"cluster_context"`
-		MaxResults     int32           `json:"max_results"`
-		TimeoutSeconds int             `json:"timeout_seconds"`
-		Attempt        int32           `json:"attempt"`
-	}
-
-	if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request body: " + err.Error()})
-		return
-	}
-	if req.Attempt < 1 {
-		req.Attempt = 1
-	}
-
-	clusterContext, err := s.parseOrBuildClusterContext(req.ClusterContext)
-	if err != nil {
-		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
-		return
-	}
-
-	timeout := normalizeTimeout(req.TimeoutSeconds, defaultPluginRunTimeout, maxPluginRunTimeout)
-	ctx, cancel := context.WithTimeout(c.Request.Context(), timeout)
-	defer cancel()
-
-	proposals, err := s.RunPluginDetection(ctx, jobType, clusterContext, req.MaxResults)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
-		return
-	}
-	detectedCount := len(proposals)
-
-	filteredProposals, skippedActiveCount, err := s.FilterPluginProposalsWithActiveJobs(jobType, proposals)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
-		return
-	}
-
-	type executionResult struct {
-		JobID      string                 `json:"job_id"`
-		Success    bool                   `json:"success"`
-		Error      string                 `json:"error,omitempty"`
-		Completion map[string]interface{} `json:"completion,omitempty"`
-	}
-
-	results := make([]executionResult, 0, len(filteredProposals))
-	successCount := 0
-	errorCount := 0
-
-	for index, proposal := range filteredProposals {
-		job := buildJobSpecFromProposal(jobType, proposal, index)
-		completed, execErr := s.ExecutePluginJob(ctx, job, clusterContext, req.Attempt)
-
-		result := executionResult{
-			JobID:   job.JobId,
-			Success: execErr == nil,
-		}
-
-		if completed != nil {
-			if payload, marshalErr := protoMessageToMap(completed); marshalErr == nil {
-				result.Completion = payload
-			}
-		}
-
-		if execErr != nil {
-			result.Error = execErr.Error()
-			errorCount++
-		} else {
-			successCount++
-		}
-
-		results = append(results, result)
-	}
-
-	c.JSON(http.StatusOK, gin.H{
-		"job_type":               jobType,
-		"detected_count":         detectedCount,
-		"ready_to_execute_count": len(filteredProposals),
-		"skipped_active_count":   skippedActiveCount,
-		"executed_count":         len(results),
-		"success_count":          successCount,
-		"error_count":            errorCount,
-		"execution_results":      results,
-	})
-}
-
-// ExecutePluginJobAPI executes one job on a capable worker and waits for completion.
-func (s *AdminServer) ExecutePluginJobAPI(c *gin.Context) {
-	var req struct {
-		Job            json.RawMessage `json:"job"`
-		ClusterContext json.RawMessage `json:"cluster_context"`
-		Attempt        int32           `json:"attempt"`
-		TimeoutSeconds int             `json:"timeout_seconds"`
-	}
-
-	if err := c.ShouldBindJSON(&req); err != nil {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request body: " + err.Error()})
-		return
-	}
-	if len(req.Job) == 0 {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "job is required"})
-		return
-	}
-
-	job := &plugin_pb.JobSpec{}
-	if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(req.Job, job); err != nil {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "invalid job payload: " + err.Error()})
-		return
-	}
-
-	clusterContext, err := s.parseOrBuildClusterContext(req.ClusterContext)
-	if err != nil {
-		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
-		return
-	}
-
-	if req.Attempt < 1 {
-		req.Attempt = 1
-	}
-
-	timeout := normalizeTimeout(req.TimeoutSeconds, defaultPluginExecutionTimeout, maxPluginExecutionTimeout)
-	ctx, cancel := context.WithTimeout(c.Request.Context(), timeout)
-	defer cancel()
-
-	completed, err := s.ExecutePluginJob(ctx, job, clusterContext, req.Attempt)
-	if err != nil {
-		if completed != nil {
-			payload, marshalErr := protoMessageToMap(completed)
-			if marshalErr == nil {
-				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error(), "completion": payload})
-				return
-			}
-		}
-		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
-		return
-	}
-
-	renderProtoJSON(c, http.StatusOK, completed)
-}
-
-func (s *AdminServer) parseOrBuildClusterContext(raw json.RawMessage) (*plugin_pb.ClusterContext, error) {
-	if len(raw) == 0 {
-		return s.buildDefaultPluginClusterContext(), nil
-	}
-
-	contextMessage := &plugin_pb.ClusterContext{}
-	if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(raw, contextMessage); err != nil {
-		return nil, fmt.Errorf("invalid cluster_context payload: %w", err)
-	}
-
-	fallback := s.buildDefaultPluginClusterContext()
-	if len(contextMessage.MasterGrpcAddresses) == 0 {
-		contextMessage.MasterGrpcAddresses = append(contextMessage.MasterGrpcAddresses, fallback.MasterGrpcAddresses...)
-	}
-	if len(contextMessage.FilerGrpcAddresses) == 0 {
-		contextMessage.FilerGrpcAddresses = append(contextMessage.FilerGrpcAddresses, fallback.FilerGrpcAddresses...)
-	}
-	if len(contextMessage.VolumeGrpcAddresses) == 0 {
-		contextMessage.VolumeGrpcAddresses = append(contextMessage.VolumeGrpcAddresses, fallback.VolumeGrpcAddresses...)
-	}
-	if contextMessage.Metadata == nil {
-		contextMessage.Metadata = map[string]string{}
-	}
-	contextMessage.Metadata["source"] = "admin"
-
-	return contextMessage, nil
-}
-
-func (s *AdminServer) buildDefaultPluginClusterContext() *plugin_pb.ClusterContext {
-	clusterContext := &plugin_pb.ClusterContext{
-		MasterGrpcAddresses: make([]string, 0),
-		FilerGrpcAddresses:  make([]string, 0),
-		VolumeGrpcAddresses: make([]string, 0),
-		Metadata: map[string]string{
-			"source": "admin",
-		},
-	}
-
-	masterAddress := string(s.masterClient.GetMaster(context.Background()))
-	if masterAddress != "" {
-		clusterContext.MasterGrpcAddresses = append(clusterContext.MasterGrpcAddresses, masterAddress)
-	}
-
-	filerSeen := map[string]struct{}{}
-	for _, filer := range s.GetAllFilers() {
-		filer = strings.TrimSpace(filer)
-		if filer == "" {
-			continue
-		}
-		if _, exists := filerSeen[filer]; exists {
-			continue
-		}
-		filerSeen[filer] = struct{}{}
-		clusterContext.FilerGrpcAddresses = append(clusterContext.FilerGrpcAddresses, filer)
-	}
-
-	volumeSeen := map[string]struct{}{}
-	if volumeServers, err := s.GetClusterVolumeServers(); err == nil {
-		for _, server := range volumeServers.VolumeServers {
-			address := strings.TrimSpace(server.GetDisplayAddress())
-			if address == "" {
-				address = strings.TrimSpace(server.Address)
-			}
-			if address == "" {
-				continue
-			}
-			if _, exists := volumeSeen[address]; exists {
-				continue
-			}
-			volumeSeen[address] = struct{}{}
-			clusterContext.VolumeGrpcAddresses = append(clusterContext.VolumeGrpcAddresses, address)
-		}
-	} else {
-		glog.V(1).Infof("failed to build default plugin volume context: %v", err)
-	}
-
-	sort.Strings(clusterContext.MasterGrpcAddresses)
-	sort.Strings(clusterContext.FilerGrpcAddresses)
-	sort.Strings(clusterContext.VolumeGrpcAddresses)
-
-	return clusterContext
-}
-
-const parseProtoJSONBodyMaxBytes = 1 << 20 // 1 MB
-
-func parseProtoJSONBody(c *gin.Context, message proto.Message) error {
-	limitedBody := http.MaxBytesReader(c.Writer, c.Request.Body, parseProtoJSONBodyMaxBytes)
-	data, err := io.ReadAll(limitedBody)
-	if err != nil {
-		return fmt.Errorf("failed to read request body: %w", err)
-	}
-	if len(data) == 0 {
-		return fmt.Errorf("request body is empty")
-	}
-	if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(data, message); err != nil {
-		return fmt.Errorf("invalid protobuf json: %w", err)
-	}
-	return nil
-}
-
-func renderProtoJSON(c *gin.Context, statusCode int, message proto.Message) {
-	payload, err := protojson.MarshalOptions{
-		UseProtoNames:   true,
-		EmitUnpopulated: true,
-	}.Marshal(message)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to encode response: " + err.Error()})
-		return
-	}
-
-	c.Data(statusCode, "application/json", payload)
-}
-
-func protoMessageToMap(message proto.Message) (map[string]interface{}, error) {
-	payload, err := protojson.MarshalOptions{UseProtoNames: true}.Marshal(message)
-	if err != nil {
-		return nil, err
-	}
-	out := map[string]interface{}{}
-	if err := json.Unmarshal(payload, &out); err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func normalizeTimeout(timeoutSeconds int, defaultTimeout, maxTimeout time.Duration) time.Duration {
-	if timeoutSeconds <= 0 {
-		return defaultTimeout
-	}
-	timeout := time.Duration(timeoutSeconds) * time.Second
-	if timeout > maxTimeout {
-		return maxTimeout
-	}
-	return timeout
-}
-
-func buildJobSpecFromProposal(jobType string, proposal *plugin_pb.JobProposal, index int) *plugin_pb.JobSpec {
-	now := timestamppb.Now()
-	suffix := make([]byte, 4)
-	if _, err := rand.Read(suffix); err != nil {
-		// Fallback to simpler ID if rand fails
-		suffix = []byte(fmt.Sprintf("%d", index))
-	}
-	jobID := fmt.Sprintf("%s-%d-%s", jobType, now.AsTime().UnixNano(), hex.EncodeToString(suffix))
-
-	jobSpec := &plugin_pb.JobSpec{
-		JobId:      jobID,
-		JobType:    jobType,
-		Priority:   plugin_pb.JobPriority_JOB_PRIORITY_NORMAL,
-		CreatedAt:  now,
-		Labels:     make(map[string]string),
-		Parameters: make(map[string]*plugin_pb.ConfigValue),
-		DedupeKey:  "",
-	}
-
-	if proposal != nil {
-		jobSpec.Summary = proposal.Summary
-		jobSpec.Detail = proposal.Detail
-		if proposal.Priority != plugin_pb.JobPriority_JOB_PRIORITY_UNSPECIFIED {
-			jobSpec.Priority = proposal.Priority
-		}
-		jobSpec.DedupeKey = proposal.DedupeKey
-		jobSpec.Parameters = plugin.CloneConfigValueMap(proposal.Parameters)
-		if proposal.Labels != nil {
-			for k, v := range proposal.Labels {
-				jobSpec.Labels[k] = v
-			}
-		}
-	}
-
-	return jobSpec
-}
-
-func parsePositiveInt(raw string, defaultValue int) int {
-	value, err := strconv.Atoi(strings.TrimSpace(raw))
-	if err != nil || value <= 0 {
-		return defaultValue
-	}
-	return value
-}
-
-// cloneConfigValueMap is now exported by the plugin package as CloneConfigValueMap

weed/admin/dash/plugin_api_test.go

@@ -1,33 +0,0 @@
-package dash
-
-import (
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-)
-
-func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
-	t.Parallel()
-
-	proposal := &plugin_pb.JobProposal{
-		ProposalId: "vacuum-2",
-		DedupeKey:  "vacuum:2",
-		JobType:    "vacuum",
-	}
-
-	jobA := buildJobSpecFromProposal("vacuum", proposal, 0)
-	jobB := buildJobSpecFromProposal("vacuum", proposal, 1)
-
-	if jobA.JobId == proposal.ProposalId {
-		t.Fatalf("job id must not reuse proposal id: %s", jobA.JobId)
-	}
-	if jobB.JobId == proposal.ProposalId {
-		t.Fatalf("job id must not reuse proposal id: %s", jobB.JobId)
-	}
-	if jobA.JobId == jobB.JobId {
-		t.Fatalf("job ids must be unique across jobs: %s", jobA.JobId)
-	}
-	if jobA.DedupeKey != proposal.DedupeKey {
-		t.Fatalf("dedupe key must be preserved: got=%s want=%s", jobA.DedupeKey, proposal.DedupeKey)
-	}
-}

weed/admin/dash/worker_grpc_server.go

@@ -5,14 +5,12 @@ import (
 	"fmt"
 	"io"
 	"net"
-	"strconv"
 	"sync"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
@@ -95,10 +93,6 @@ func (s *WorkerGrpcServer) StartWithTLS(port int) error {
 	grpcServer := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.admin"))
 
 	worker_pb.RegisterWorkerServiceServer(grpcServer, s)
-	if plugin := s.adminServer.GetPlugin(); plugin != nil {
-		plugin_pb.RegisterPluginControlServiceServer(grpcServer, plugin)
-		glog.V(0).Infof("Plugin gRPC service registered on worker gRPC server")
-	}
 
 	s.grpcServer = grpcServer
 	s.listener = listener
@@ -120,25 +114,6 @@ func (s *WorkerGrpcServer) StartWithTLS(port int) error {
 	return nil
 }
 
-// ListenPort returns the currently bound worker gRPC listen port.
-func (s *WorkerGrpcServer) ListenPort() int {
-	if s == nil || s.listener == nil {
-		return 0
-	}
-	if tcpAddr, ok := s.listener.Addr().(*net.TCPAddr); ok {
-		return tcpAddr.Port
-	}
-	_, portStr, err := net.SplitHostPort(s.listener.Addr().String())
-	if err != nil {
-		return 0
-	}
-	port, err := strconv.Atoi(portStr)
-	if err != nil {
-		return 0
-	}
-	return port
-}
-
 // Stop stops the gRPC server
 func (s *WorkerGrpcServer) Stop() error {
 	if !s.running {

weed/admin/handlers/admin_handlers.go

@@ -23,7 +23,7 @@ type AdminHandlers struct {
 	fileBrowserHandlers    *FileBrowserHandlers
 	userHandlers           *UserHandlers
 	policyHandlers         *PolicyHandlers
-	pluginHandlers         *PluginHandlers
+	maintenanceHandlers    *MaintenanceHandlers
 	mqHandlers             *MessageQueueHandlers
 	serviceAccountHandlers *ServiceAccountHandlers
 }
@@ -35,7 +35,7 @@ func NewAdminHandlers(adminServer *dash.AdminServer) *AdminHandlers {
 	fileBrowserHandlers := NewFileBrowserHandlers(adminServer)
 	userHandlers := NewUserHandlers(adminServer)
 	policyHandlers := NewPolicyHandlers(adminServer)
-	pluginHandlers := NewPluginHandlers(adminServer)
+	maintenanceHandlers := NewMaintenanceHandlers(adminServer)
 	mqHandlers := NewMessageQueueHandlers(adminServer)
 	serviceAccountHandlers := NewServiceAccountHandlers(adminServer)
 	return &AdminHandlers{
@@ -45,7 +45,7 @@ func NewAdminHandlers(adminServer *dash.AdminServer) *AdminHandlers {
 		fileBrowserHandlers:    fileBrowserHandlers,
 		userHandlers:           userHandlers,
 		policyHandlers:         policyHandlers,
-		pluginHandlers:         pluginHandlers,
+		maintenanceHandlers:    maintenanceHandlers,
 		mqHandlers:             mqHandlers,
 		serviceAccountHandlers: serviceAccountHandlers,
 	}
@@ -119,12 +119,14 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
 		protected.GET("/mq/topics", h.mqHandlers.ShowTopics)
 		protected.GET("/mq/topics/:namespace/:topic", h.mqHandlers.ShowTopicDetails)
 
-		protected.GET("/plugin", h.pluginHandlers.ShowPlugin)
-		protected.GET("/plugin/configuration", h.pluginHandlers.ShowPluginConfiguration)
-		protected.GET("/plugin/queue", h.pluginHandlers.ShowPluginQueue)
-		protected.GET("/plugin/detection", h.pluginHandlers.ShowPluginDetection)
-		protected.GET("/plugin/execution", h.pluginHandlers.ShowPluginExecution)
-		protected.GET("/plugin/monitoring", h.pluginHandlers.ShowPluginMonitoring)
+		// Maintenance system routes
+		protected.GET("/maintenance", h.maintenanceHandlers.ShowMaintenanceQueue)
+		protected.GET("/maintenance/workers", h.maintenanceHandlers.ShowMaintenanceWorkers)
+		protected.GET("/maintenance/config", h.maintenanceHandlers.ShowMaintenanceConfig)
+		protected.POST("/maintenance/config", dash.RequireWriteAccess(), h.maintenanceHandlers.UpdateMaintenanceConfig)
+		protected.GET("/maintenance/config/:taskType", h.maintenanceHandlers.ShowTaskConfig)
+		protected.POST("/maintenance/config/:taskType", dash.RequireWriteAccess(), h.maintenanceHandlers.UpdateTaskConfig)
+		protected.GET("/maintenance/tasks/:id", h.maintenanceHandlers.ShowTaskDetail)
 
 		// API routes for AJAX calls
 		api := r.Group("/api")
@@ -224,25 +226,20 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
 				volumeApi.POST("/:id/:server/vacuum", dash.RequireWriteAccess(), h.clusterHandlers.VacuumVolume)
 			}
 
-			// Plugin API routes
-			pluginApi := api.Group("/plugin")
+			// Maintenance API routes
+			maintenanceApi := api.Group("/maintenance")
 			{
-				pluginApi.GET("/status", h.adminServer.GetPluginStatusAPI)
-				pluginApi.GET("/workers", h.adminServer.GetPluginWorkersAPI)
-				pluginApi.GET("/job-types", h.adminServer.GetPluginJobTypesAPI)
-				pluginApi.GET("/jobs", h.adminServer.GetPluginJobsAPI)
-				pluginApi.GET("/jobs/:jobId", h.adminServer.GetPluginJobAPI)
-				pluginApi.GET("/jobs/:jobId/detail", h.adminServer.GetPluginJobDetailAPI)
-				pluginApi.GET("/activities", h.adminServer.GetPluginActivitiesAPI)
-				pluginApi.GET("/scheduler-states", h.adminServer.GetPluginSchedulerStatesAPI)
-				pluginApi.GET("/job-types/:jobType/descriptor", h.adminServer.GetPluginJobTypeDescriptorAPI)
-				pluginApi.POST("/job-types/:jobType/schema", h.adminServer.RequestPluginJobTypeSchemaAPI)
-				pluginApi.GET("/job-types/:jobType/config", h.adminServer.GetPluginJobTypeConfigAPI)
-				pluginApi.PUT("/job-types/:jobType/config", dash.RequireWriteAccess(), h.adminServer.UpdatePluginJobTypeConfigAPI)
-				pluginApi.GET("/job-types/:jobType/runs", h.adminServer.GetPluginRunHistoryAPI)
-				pluginApi.POST("/job-types/:jobType/detect", dash.RequireWriteAccess(), h.adminServer.TriggerPluginDetectionAPI)
-				pluginApi.POST("/job-types/:jobType/run", dash.RequireWriteAccess(), h.adminServer.RunPluginJobTypeAPI)
-				pluginApi.POST("/jobs/execute", dash.RequireWriteAccess(), h.adminServer.ExecutePluginJobAPI)
+				maintenanceApi.POST("/scan", dash.RequireWriteAccess(), h.adminServer.TriggerMaintenanceScan)
+				maintenanceApi.GET("/tasks", h.adminServer.GetMaintenanceTasks)
+				maintenanceApi.GET("/tasks/:id", h.adminServer.GetMaintenanceTask)
+				maintenanceApi.GET("/tasks/:id/detail", h.adminServer.GetMaintenanceTaskDetailAPI)
+				maintenanceApi.POST("/tasks/:id/cancel", dash.RequireWriteAccess(), h.adminServer.CancelMaintenanceTask)
+				maintenanceApi.GET("/workers", h.adminServer.GetMaintenanceWorkersAPI)
+				maintenanceApi.GET("/workers/:id", h.adminServer.GetMaintenanceWorker)
+				maintenanceApi.GET("/workers/:id/logs", h.adminServer.GetWorkerLogs)
+				maintenanceApi.GET("/stats", h.adminServer.GetMaintenanceStats)
+				maintenanceApi.GET("/config", h.adminServer.GetMaintenanceConfigAPI)
+				maintenanceApi.PUT("/config", dash.RequireWriteAccess(), h.adminServer.UpdateMaintenanceConfigAPI)
 			}
 
 			// Message Queue API routes
@@ -295,12 +292,14 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
 		r.GET("/mq/topics", h.mqHandlers.ShowTopics)
 		r.GET("/mq/topics/:namespace/:topic", h.mqHandlers.ShowTopicDetails)
 
-		r.GET("/plugin", h.pluginHandlers.ShowPlugin)
-		r.GET("/plugin/configuration", h.pluginHandlers.ShowPluginConfiguration)
-		r.GET("/plugin/queue", h.pluginHandlers.ShowPluginQueue)
-		r.GET("/plugin/detection", h.pluginHandlers.ShowPluginDetection)
-		r.GET("/plugin/execution", h.pluginHandlers.ShowPluginExecution)
-		r.GET("/plugin/monitoring", h.pluginHandlers.ShowPluginMonitoring)
+		// Maintenance system routes
+		r.GET("/maintenance", h.maintenanceHandlers.ShowMaintenanceQueue)
+		r.GET("/maintenance/workers", h.maintenanceHandlers.ShowMaintenanceWorkers)
+		r.GET("/maintenance/config", h.maintenanceHandlers.ShowMaintenanceConfig)
+		r.POST("/maintenance/config", h.maintenanceHandlers.UpdateMaintenanceConfig)
+		r.GET("/maintenance/config/:taskType", h.maintenanceHandlers.ShowTaskConfig)
+		r.POST("/maintenance/config/:taskType", h.maintenanceHandlers.UpdateTaskConfig)
+		r.GET("/maintenance/tasks/:id", h.maintenanceHandlers.ShowTaskDetail)
 
 		// API routes for AJAX calls
 		api := r.Group("/api")
@@ -399,25 +398,20 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
 				volumeApi.POST("/:id/:server/vacuum", h.clusterHandlers.VacuumVolume)
 			}
 
-			// Plugin API routes
-			pluginApi := api.Group("/plugin")
+			// Maintenance API routes
+			maintenanceApi := api.Group("/maintenance")
 			{
-				pluginApi.GET("/status", h.adminServer.GetPluginStatusAPI)
-				pluginApi.GET("/workers", h.adminServer.GetPluginWorkersAPI)
-				pluginApi.GET("/job-types", h.adminServer.GetPluginJobTypesAPI)
-				pluginApi.GET("/jobs", h.adminServer.GetPluginJobsAPI)
-				pluginApi.GET("/jobs/:jobId", h.adminServer.GetPluginJobAPI)
-				pluginApi.GET("/jobs/:jobId/detail", h.adminServer.GetPluginJobDetailAPI)
-				pluginApi.GET("/activities", h.adminServer.GetPluginActivitiesAPI)
-				pluginApi.GET("/scheduler-states", h.adminServer.GetPluginSchedulerStatesAPI)
-				pluginApi.GET("/job-types/:jobType/descriptor", h.adminServer.GetPluginJobTypeDescriptorAPI)
-				pluginApi.POST("/job-types/:jobType/schema", h.adminServer.RequestPluginJobTypeSchemaAPI)
-				pluginApi.GET("/job-types/:jobType/config", h.adminServer.GetPluginJobTypeConfigAPI)
-				pluginApi.PUT("/job-types/:jobType/config", h.adminServer.UpdatePluginJobTypeConfigAPI)
-				pluginApi.GET("/job-types/:jobType/runs", h.adminServer.GetPluginRunHistoryAPI)
-				pluginApi.POST("/job-types/:jobType/detect", h.adminServer.TriggerPluginDetectionAPI)
-				pluginApi.POST("/job-types/:jobType/run", h.adminServer.RunPluginJobTypeAPI)
-				pluginApi.POST("/jobs/execute", h.adminServer.ExecutePluginJobAPI)
+				maintenanceApi.POST("/scan", h.adminServer.TriggerMaintenanceScan)
+				maintenanceApi.GET("/tasks", h.adminServer.GetMaintenanceTasks)
+				maintenanceApi.GET("/tasks/:id", h.adminServer.GetMaintenanceTask)
+				maintenanceApi.GET("/tasks/:id/detail", h.adminServer.GetMaintenanceTaskDetailAPI)
+				maintenanceApi.POST("/tasks/:id/cancel", h.adminServer.CancelMaintenanceTask)
+				maintenanceApi.GET("/workers", h.adminServer.GetMaintenanceWorkersAPI)
+				maintenanceApi.GET("/workers/:id", h.adminServer.GetMaintenanceWorker)
+				maintenanceApi.GET("/workers/:id/logs", h.adminServer.GetWorkerLogs)
+				maintenanceApi.GET("/stats", h.adminServer.GetMaintenanceStats)
+				maintenanceApi.GET("/config", h.adminServer.GetMaintenanceConfigAPI)
+				maintenanceApi.PUT("/config", h.adminServer.UpdateMaintenanceConfigAPI)
 			}
 
 			// Message Queue API routes

weed/admin/handlers/admin_handlers_routes_test.go

@@ -1,95 +0,0 @@
-package handlers
-
-import (
-	"testing"
-
-	"github.com/gin-gonic/gin"
-	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
-)
-
-func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_NoAuth(t *testing.T) {
-	gin.SetMode(gin.TestMode)
-	router := gin.New()
-
-	newRouteTestAdminHandlers().SetupRoutes(router, false, "", "", "", "", true)
-
-	if !hasRoute(router, "GET", "/api/plugin/scheduler-states") {
-		t.Fatalf("expected GET /api/plugin/scheduler-states to be registered in no-auth mode")
-	}
-	if !hasRoute(router, "GET", "/api/plugin/jobs/:jobId/detail") {
-		t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in no-auth mode")
-	}
-}
-
-func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_WithAuth(t *testing.T) {
-	gin.SetMode(gin.TestMode)
-	router := gin.New()
-
-	newRouteTestAdminHandlers().SetupRoutes(router, true, "admin", "password", "", "", true)
-
-	if !hasRoute(router, "GET", "/api/plugin/scheduler-states") {
-		t.Fatalf("expected GET /api/plugin/scheduler-states to be registered in auth mode")
-	}
-	if !hasRoute(router, "GET", "/api/plugin/jobs/:jobId/detail") {
-		t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in auth mode")
-	}
-}
-
-func TestSetupRoutes_RegistersPluginPages_NoAuth(t *testing.T) {
-	gin.SetMode(gin.TestMode)
-	router := gin.New()
-
-	newRouteTestAdminHandlers().SetupRoutes(router, false, "", "", "", "", true)
-
-	assertHasRoute(t, router, "GET", "/plugin")
-	assertHasRoute(t, router, "GET", "/plugin/configuration")
-	assertHasRoute(t, router, "GET", "/plugin/queue")
-	assertHasRoute(t, router, "GET", "/plugin/detection")
-	assertHasRoute(t, router, "GET", "/plugin/execution")
-	assertHasRoute(t, router, "GET", "/plugin/monitoring")
-}
-
-func TestSetupRoutes_RegistersPluginPages_WithAuth(t *testing.T) {
-	gin.SetMode(gin.TestMode)
-	router := gin.New()
-
-	newRouteTestAdminHandlers().SetupRoutes(router, true, "admin", "password", "", "", true)
-
-	assertHasRoute(t, router, "GET", "/plugin")
-	assertHasRoute(t, router, "GET", "/plugin/configuration")
-	assertHasRoute(t, router, "GET", "/plugin/queue")
-	assertHasRoute(t, router, "GET", "/plugin/detection")
-	assertHasRoute(t, router, "GET", "/plugin/execution")
-	assertHasRoute(t, router, "GET", "/plugin/monitoring")
-}
-
-func newRouteTestAdminHandlers() *AdminHandlers {
-	adminServer := &dash.AdminServer{}
-	return &AdminHandlers{
-		adminServer:            adminServer,
-		authHandlers:           &AuthHandlers{adminServer: adminServer},
-		clusterHandlers:        &ClusterHandlers{adminServer: adminServer},
-		fileBrowserHandlers:    &FileBrowserHandlers{adminServer: adminServer},
-		userHandlers:           &UserHandlers{adminServer: adminServer},
-		policyHandlers:         &PolicyHandlers{adminServer: adminServer},
-		pluginHandlers:         &PluginHandlers{adminServer: adminServer},
-		mqHandlers:             &MessageQueueHandlers{adminServer: adminServer},
-		serviceAccountHandlers: &ServiceAccountHandlers{adminServer: adminServer},
-	}
-}
-
-func hasRoute(router *gin.Engine, method string, path string) bool {
-	for _, route := range router.Routes() {
-		if route.Method == method && route.Path == path {
-			return true
-		}
-	}
-	return false
-}
-
-func assertHasRoute(t *testing.T, router *gin.Engine, method string, path string) {
-	t.Helper()
-	if !hasRoute(router, method, path) {
-		t.Fatalf("expected %s %s to be registered", method, path)
-	}
-}

weed/admin/handlers/maintenance_handlers.go

@@ -0,0 +1,550 @@
+package handlers
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/seaweedfs/seaweedfs/weed/admin/config"
+	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
+	"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
+	"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
+	"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
+	"github.com/seaweedfs/seaweedfs/weed/worker/types"
+)
+
+// MaintenanceHandlers handles maintenance-related HTTP requests
+type MaintenanceHandlers struct {
+	adminServer *dash.AdminServer
+}
+
+// NewMaintenanceHandlers creates a new instance of MaintenanceHandlers
+func NewMaintenanceHandlers(adminServer *dash.AdminServer) *MaintenanceHandlers {
+	return &MaintenanceHandlers{
+		adminServer: adminServer,
+	}
+}
+
+// ShowTaskDetail displays the task detail page
+func (h *MaintenanceHandlers) ShowTaskDetail(c *gin.Context) {
+	taskID := c.Param("id")
+
+	if h.adminServer == nil {
+		c.String(http.StatusInternalServerError, "Admin server not initialized")
+		return
+	}
+
+	taskDetail, err := h.adminServer.GetMaintenanceTaskDetail(taskID)
+	if err != nil {
+		glog.Errorf("DEBUG ShowTaskDetail: error getting task detail for %s: %v", taskID, err)
+		c.String(http.StatusNotFound, "Task not found: %s (Error: %v)", taskID, err)
+		return
+	}
+
+	c.Header("Content-Type", "text/html")
+	taskDetailComponent := app.TaskDetail(taskDetail)
+	layoutComponent := layout.Layout(c, taskDetailComponent)
+	err = layoutComponent.Render(c.Request.Context(), c.Writer)
+	if err != nil {
+		glog.Errorf("DEBUG ShowTaskDetail: render error: %v", err)
+		c.String(http.StatusInternalServerError, "Failed to render template: %v", err)
+		return
+	}
+
+}
+
+// ShowMaintenanceQueue displays the maintenance queue page
+func (h *MaintenanceHandlers) ShowMaintenanceQueue(c *gin.Context) {
+	// Add timeout to prevent hanging
+	ctx, cancel := context.WithTimeout(c.Request.Context(), 30*time.Second)
+	defer cancel()
+
+	// Use a channel to handle timeout for data retrieval
+	type result struct {
+		data *maintenance.MaintenanceQueueData
+		err  error
+	}
+	resultChan := make(chan result, 1)
+
+	go func() {
+		data, err := h.getMaintenanceQueueData()
+		resultChan <- result{data: data, err: err}
+	}()
+
+	select {
+	case res := <-resultChan:
+		if res.err != nil {
+			glog.V(1).Infof("ShowMaintenanceQueue: error getting data: %v", res.err)
+			c.JSON(http.StatusInternalServerError, gin.H{"error": res.err.Error()})
+			return
+		}
+
+		glog.V(2).Infof("ShowMaintenanceQueue: got data with %d tasks", len(res.data.Tasks))
+
+		// Render HTML template
+		c.Header("Content-Type", "text/html")
+		maintenanceComponent := app.MaintenanceQueue(res.data)
+		layoutComponent := layout.Layout(c, maintenanceComponent)
+		err := layoutComponent.Render(ctx, c.Writer)
+		if err != nil {
+			glog.V(1).Infof("ShowMaintenanceQueue: render error: %v", err)
+			c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
+			return
+		}
+
+		glog.V(3).Infof("ShowMaintenanceQueue: template rendered successfully")
+
+	case <-ctx.Done():
+		glog.Warningf("ShowMaintenanceQueue: timeout waiting for data")
+		c.JSON(http.StatusRequestTimeout, gin.H{
+			"error":      "Request timeout - maintenance data retrieval took too long. This may indicate a system issue.",
+			"suggestion": "Try refreshing the page or contact system administrator if the problem persists.",
+		})
+		return
+	}
+}
+
+// ShowMaintenanceWorkers displays the maintenance workers page
+func (h *MaintenanceHandlers) ShowMaintenanceWorkers(c *gin.Context) {
+	if h.adminServer == nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Admin server not initialized"})
+		return
+	}
+	workersData, err := h.adminServer.GetMaintenanceWorkersData()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	// Render HTML template
+	c.Header("Content-Type", "text/html")
+	workersComponent := app.MaintenanceWorkers(workersData)
+	layoutComponent := layout.Layout(c, workersComponent)
+	err = layoutComponent.Render(c.Request.Context(), c.Writer)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
+		return
+	}
+}
+
+// ShowMaintenanceConfig displays the maintenance configuration page
+func (h *MaintenanceHandlers) ShowMaintenanceConfig(c *gin.Context) {
+	config, err := h.getMaintenanceConfig()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	// Get the schema for dynamic form rendering
+	schema := maintenance.GetMaintenanceConfigSchema()
+
+	// Render HTML template using schema-driven approach
+	c.Header("Content-Type", "text/html")
+	configComponent := app.MaintenanceConfigSchema(config, schema)
+	layoutComponent := layout.Layout(c, configComponent)
+	err = layoutComponent.Render(c.Request.Context(), c.Writer)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
+		return
+	}
+}
+
+// ShowTaskConfig displays the configuration page for a specific task type
+func (h *MaintenanceHandlers) ShowTaskConfig(c *gin.Context) {
+	taskTypeName := c.Param("taskType")
+
+	// Get the schema for this task type
+	schema := tasks.GetTaskConfigSchema(taskTypeName)
+	if schema == nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "Task type not found or no schema available"})
+		return
+	}
+
+	// Get the UI provider for current configuration
+	uiRegistry := tasks.GetGlobalUIRegistry()
+	typesRegistry := tasks.GetGlobalTypesRegistry()
+
+	var provider types.TaskUIProvider
+	for workerTaskType := range typesRegistry.GetAllDetectors() {
+		if string(workerTaskType) == taskTypeName {
+			provider = uiRegistry.GetProvider(workerTaskType)
+			break
+		}
+	}
+
+	if provider == nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "UI provider not found for task type"})
+		return
+	}
+
+	// Get current configuration
+	currentConfig := provider.GetCurrentConfig()
+
+	// Note: Do NOT apply schema defaults to current config as it overrides saved values
+	// Only apply defaults when creating new configs, not when displaying existing ones
+
+	// Create task configuration data
+	configData := &maintenance.TaskConfigData{
+		TaskType:    maintenance.MaintenanceTaskType(taskTypeName),
+		TaskName:    schema.DisplayName,
+		TaskIcon:    schema.Icon,
+		Description: schema.Description,
+	}
+
+	// Render HTML template using schema-based approach
+	c.Header("Content-Type", "text/html")
+	taskConfigComponent := app.TaskConfigSchema(configData, schema, currentConfig)
+	layoutComponent := layout.Layout(c, taskConfigComponent)
+	err := layoutComponent.Render(c.Request.Context(), c.Writer)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
+		return
+	}
+}
+
+// UpdateTaskConfig updates task configuration from form
+func (h *MaintenanceHandlers) UpdateTaskConfig(c *gin.Context) {
+	taskTypeName := c.Param("taskType")
+	taskType := types.TaskType(taskTypeName)
+
+	// Parse form data
+	err := c.Request.ParseForm()
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to parse form data: " + err.Error()})
+		return
+	}
+
+	// Debug logging - show received form data
+	glog.V(1).Infof("Received form data for task type %s:", taskTypeName)
+	for key, values := range c.Request.PostForm {
+		glog.V(1).Infof("  %s: %v", key, values)
+	}
+
+	// Get the task configuration schema
+	schema := tasks.GetTaskConfigSchema(taskTypeName)
+	if schema == nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "Schema not found for task type: " + taskTypeName})
+		return
+	}
+
+	// Create a new config instance based on task type and apply schema defaults
+	var config TaskConfig
+	switch taskType {
+	case types.TaskTypeVacuum:
+		config = &vacuum.Config{}
+	case types.TaskTypeBalance:
+		config = &balance.Config{}
+	case types.TaskTypeErasureCoding:
+		config = &erasure_coding.Config{}
+	default:
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Unsupported task type: " + taskTypeName})
+		return
+	}
+
+	// Apply schema defaults first using type-safe method
+	if err := schema.ApplyDefaultsToConfig(config); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to apply defaults: " + err.Error()})
+		return
+	}
+
+	// First, get the current configuration to preserve existing values
+	currentUIRegistry := tasks.GetGlobalUIRegistry()
+	currentTypesRegistry := tasks.GetGlobalTypesRegistry()
+
+	var currentProvider types.TaskUIProvider
+	for workerTaskType := range currentTypesRegistry.GetAllDetectors() {
+		if string(workerTaskType) == string(taskType) {
+			currentProvider = currentUIRegistry.GetProvider(workerTaskType)
+			break
+		}
+	}
+
+	if currentProvider != nil {
+		// Copy current config values to the new config
+		currentConfig := currentProvider.GetCurrentConfig()
+		if currentConfigProtobuf, ok := currentConfig.(TaskConfig); ok {
+			// Apply current values using protobuf directly - no map conversion needed!
+			currentPolicy := currentConfigProtobuf.ToTaskPolicy()
+			if err := config.FromTaskPolicy(currentPolicy); err != nil {
+				glog.Warningf("Failed to load current config for %s: %v", taskTypeName, err)
+			}
+		}
+	}
+
+	// Parse form data using schema-based approach (this will override with new values)
+	err = h.parseTaskConfigFromForm(c.Request.PostForm, schema, config)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to parse configuration: " + err.Error()})
+		return
+	}
+
+	// Debug logging - show parsed config values
+	switch taskType {
+	case types.TaskTypeVacuum:
+		if vacuumConfig, ok := config.(*vacuum.Config); ok {
+			glog.V(1).Infof("Parsed vacuum config - GarbageThreshold: %f, MinVolumeAgeSeconds: %d, MinIntervalSeconds: %d",
+				vacuumConfig.GarbageThreshold, vacuumConfig.MinVolumeAgeSeconds, vacuumConfig.MinIntervalSeconds)
+		}
+	case types.TaskTypeErasureCoding:
+		if ecConfig, ok := config.(*erasure_coding.Config); ok {
+			glog.V(1).Infof("Parsed EC config - FullnessRatio: %f, QuietForSeconds: %d, MinSizeMB: %d, CollectionFilter: '%s'",
+				ecConfig.FullnessRatio, ecConfig.QuietForSeconds, ecConfig.MinSizeMB, ecConfig.CollectionFilter)
+		}
+	case types.TaskTypeBalance:
+		if balanceConfig, ok := config.(*balance.Config); ok {
+			glog.V(1).Infof("Parsed balance config - Enabled: %v, MaxConcurrent: %d, ScanIntervalSeconds: %d, ImbalanceThreshold: %f, MinServerCount: %d",
+				balanceConfig.Enabled, balanceConfig.MaxConcurrent, balanceConfig.ScanIntervalSeconds, balanceConfig.ImbalanceThreshold, balanceConfig.MinServerCount)
+		}
+	}
+
+	// Validate the configuration
+	if validationErrors := schema.ValidateConfig(config); len(validationErrors) > 0 {
+		errorMessages := make([]string, len(validationErrors))
+		for i, err := range validationErrors {
+			errorMessages[i] = err.Error()
+		}
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Configuration validation failed", "details": errorMessages})
+		return
+	}
+
+	// Apply configuration using UIProvider
+	uiRegistry := tasks.GetGlobalUIRegistry()
+	typesRegistry := tasks.GetGlobalTypesRegistry()
+
+	var provider types.TaskUIProvider
+	for workerTaskType := range typesRegistry.GetAllDetectors() {
+		if string(workerTaskType) == string(taskType) {
+			provider = uiRegistry.GetProvider(workerTaskType)
+			break
+		}
+	}
+
+	if provider == nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "UI provider not found for task type"})
+		return
+	}
+
+	// Apply configuration using provider
+	err = provider.ApplyTaskConfig(config)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to apply configuration: " + err.Error()})
+		return
+	}
+
+	// Save task configuration to protobuf file using ConfigPersistence
+	if h.adminServer != nil && h.adminServer.GetConfigPersistence() != nil {
+		err = h.saveTaskConfigToProtobuf(taskType, config)
+		if err != nil {
+			glog.Warningf("Failed to save task config to protobuf file: %v", err)
+			// Don't fail the request, just log the warning
+		}
+	} else if h.adminServer == nil {
+		glog.Warningf("Failed to save task config: admin server not initialized")
+	}
+
+	// Trigger a configuration reload in the maintenance manager
+	if h.adminServer != nil {
+		if manager := h.adminServer.GetMaintenanceManager(); manager != nil {
+			err = manager.ReloadTaskConfigurations()
+			if err != nil {
+				glog.Warningf("Failed to reload task configurations: %v", err)
+			} else {
+				glog.V(1).Infof("Successfully reloaded task configurations after updating %s", taskTypeName)
+			}
+		}
+	}
+
+	// Redirect back to task configuration page
+	c.Redirect(http.StatusSeeOther, "/maintenance/config/"+taskTypeName)
+}
+
+// parseTaskConfigFromForm parses form data using schema definitions
+func (h *MaintenanceHandlers) parseTaskConfigFromForm(formData map[string][]string, schema *tasks.TaskConfigSchema, config interface{}) error {
+	configValue := reflect.ValueOf(config)
+	if configValue.Kind() == reflect.Ptr {
+		configValue = configValue.Elem()
+	}
+
+	if configValue.Kind() != reflect.Struct {
+		return fmt.Errorf("config must be a struct or pointer to struct")
+	}
+
+	configType := configValue.Type()
+
+	for i := 0; i < configValue.NumField(); i++ {
+		field := configValue.Field(i)
+		fieldType := configType.Field(i)
+
+		// Handle embedded structs recursively
+		if fieldType.Anonymous && field.Kind() == reflect.Struct {
+			err := h.parseTaskConfigFromForm(formData, schema, field.Addr().Interface())
+			if err != nil {
+				return fmt.Errorf("error parsing embedded struct %s: %w", fieldType.Name, err)
+			}
+			continue
+		}
+
+		// Get JSON tag name
+		jsonTag := fieldType.Tag.Get("json")
+		if jsonTag == "" {
+			continue
+		}
+
+		// Remove options like ",omitempty"
+		if commaIdx := strings.Index(jsonTag, ","); commaIdx > 0 {
+			jsonTag = jsonTag[:commaIdx]
+		}
+
+		// Find corresponding schema field
+		schemaField := schema.GetFieldByName(jsonTag)
+		if schemaField == nil {
+			continue
+		}
+
+		// Parse value based on field type
+		if err := h.parseFieldFromForm(formData, schemaField, field); err != nil {
+			return fmt.Errorf("error parsing field %s: %w", schemaField.DisplayName, err)
+		}
+	}
+
+	return nil
+}
+
+// parseFieldFromForm parses a single field value from form data
+func (h *MaintenanceHandlers) parseFieldFromForm(formData map[string][]string, schemaField *config.Field, fieldValue reflect.Value) error {
+	if !fieldValue.CanSet() {
+		return nil
+	}
+
+	switch schemaField.Type {
+	case config.FieldTypeBool:
+		// Checkbox fields - present means true, absent means false
+		_, exists := formData[schemaField.JSONName]
+		fieldValue.SetBool(exists)
+
+	case config.FieldTypeInt:
+		if values, ok := formData[schemaField.JSONName]; ok && len(values) > 0 {
+			if intVal, err := strconv.Atoi(values[0]); err != nil {
+				return fmt.Errorf("invalid integer value: %s", values[0])
+			} else {
+				fieldValue.SetInt(int64(intVal))
+			}
+		}
+
+	case config.FieldTypeFloat:
+		if values, ok := formData[schemaField.JSONName]; ok && len(values) > 0 {
+			if floatVal, err := strconv.ParseFloat(values[0], 64); err != nil {
+				return fmt.Errorf("invalid float value: %s", values[0])
+			} else {
+				fieldValue.SetFloat(floatVal)
+			}
+		}
+
+	case config.FieldTypeString:
+		if values, ok := formData[schemaField.JSONName]; ok && len(values) > 0 {
+			fieldValue.SetString(values[0])
+		}
+
+	case config.FieldTypeInterval:
+		// Parse interval fields with value + unit
+		valueKey := schemaField.JSONName + "_value"
+		unitKey := schemaField.JSONName + "_unit"
+
+		if valueStrs, ok := formData[valueKey]; ok && len(valueStrs) > 0 {
+			value, err := strconv.Atoi(valueStrs[0])
+			if err != nil {
+				return fmt.Errorf("invalid interval value: %s", valueStrs[0])
+			}
+
+			unit := "minutes" // default
+			if unitStrs, ok := formData[unitKey]; ok && len(unitStrs) > 0 {
+				unit = unitStrs[0]
+			}
+
+			// Convert to seconds
+			seconds := config.IntervalValueUnitToSeconds(value, unit)
+			fieldValue.SetInt(int64(seconds))
+		}
+
+	default:
+		return fmt.Errorf("unsupported field type: %s", schemaField.Type)
+	}
+
+	return nil
+}
+
+// UpdateMaintenanceConfig updates maintenance configuration from form
+func (h *MaintenanceHandlers) UpdateMaintenanceConfig(c *gin.Context) {
+	var config maintenance.MaintenanceConfig
+	if err := c.ShouldBind(&config); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	err := h.updateMaintenanceConfig(&config)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	c.Redirect(http.StatusSeeOther, "/maintenance/config")
+}
+
+// Helper methods that delegate to AdminServer
+
+func (h *MaintenanceHandlers) getMaintenanceQueueData() (*maintenance.MaintenanceQueueData, error) {
+	if h.adminServer == nil {
+		return nil, fmt.Errorf("admin server not initialized")
+	}
+	// Use the exported method from AdminServer used by the JSON API
+	return h.adminServer.GetMaintenanceQueueData()
+}
+
+func (h *MaintenanceHandlers) getMaintenanceConfig() (*maintenance.MaintenanceConfigData, error) {
+	if h.adminServer == nil {
+		return nil, fmt.Errorf("admin server not initialized")
+	}
+	// Delegate to AdminServer's real persistence method
+	return h.adminServer.GetMaintenanceConfigData()
+}
+
+func (h *MaintenanceHandlers) updateMaintenanceConfig(config *maintenance.MaintenanceConfig) error {
+	if h.adminServer == nil {
+		return fmt.Errorf("admin server not initialized")
+	}
+	// Delegate to AdminServer's real persistence method
+	return h.adminServer.UpdateMaintenanceConfigData(config)
+}
+
+// saveTaskConfigToProtobuf saves task configuration to protobuf file
+func (h *MaintenanceHandlers) saveTaskConfigToProtobuf(taskType types.TaskType, config TaskConfig) error {
+	configPersistence := h.adminServer.GetConfigPersistence()
+	if configPersistence == nil {
+		return fmt.Errorf("config persistence not available")
+	}
+
+	// Use the new ToTaskPolicy method - much simpler and more maintainable!
+	taskPolicy := config.ToTaskPolicy()
+
+	// Save using task-specific methods
+	switch taskType {
+	case types.TaskTypeVacuum:
+		return configPersistence.SaveVacuumTaskPolicy(taskPolicy)
+	case types.TaskTypeErasureCoding:
+		return configPersistence.SaveErasureCodingTaskPolicy(taskPolicy)
+	case types.TaskTypeBalance:
+		return configPersistence.SaveBalanceTaskPolicy(taskPolicy)
+	default:
+		return fmt.Errorf("unsupported task type for protobuf persistence: %s", taskType)
+	}
+}

weed/admin/handlers/maintenance_handlers_test.go

@@ -0,0 +1,389 @@
+package handlers
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/admin/config"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
+)
+
+func TestParseTaskConfigFromForm_WithEmbeddedStruct(t *testing.T) {
+	// Create a maintenance handlers instance for testing
+	h := &MaintenanceHandlers{}
+
+	// Test with balance config
+	t.Run("Balance Config", func(t *testing.T) {
+		// Simulate form data
+		formData := url.Values{
+			"enabled":                     {"on"},      // checkbox field
+			"scan_interval_seconds_value": {"30"},      // interval field
+			"scan_interval_seconds_unit":  {"minutes"}, // interval unit
+			"max_concurrent":              {"2"},       // number field
+			"imbalance_threshold":         {"0.15"},    // float field
+			"min_server_count":            {"3"},       // number field
+		}
+
+		// Get schema
+		schema := tasks.GetTaskConfigSchema("balance")
+		if schema == nil {
+			t.Fatal("Failed to get balance schema")
+		}
+
+		// Create config instance
+		config := &balance.Config{}
+
+		// Parse form data
+		err := h.parseTaskConfigFromForm(formData, schema, config)
+		if err != nil {
+			t.Fatalf("Failed to parse form data: %v", err)
+		}
+
+		// Verify embedded struct fields were set correctly
+		if !config.Enabled {
+			t.Errorf("Expected Enabled=true, got %v", config.Enabled)
+		}
+
+		if config.ScanIntervalSeconds != 1800 { // 30 minutes * 60
+			t.Errorf("Expected ScanIntervalSeconds=1800, got %v", config.ScanIntervalSeconds)
+		}
+
+		if config.MaxConcurrent != 2 {
+			t.Errorf("Expected MaxConcurrent=2, got %v", config.MaxConcurrent)
+		}
+
+		// Verify balance-specific fields were set correctly
+		if config.ImbalanceThreshold != 0.15 {
+			t.Errorf("Expected ImbalanceThreshold=0.15, got %v", config.ImbalanceThreshold)
+		}
+
+		if config.MinServerCount != 3 {
+			t.Errorf("Expected MinServerCount=3, got %v", config.MinServerCount)
+		}
+	})
+
+	// Test with vacuum config
+	t.Run("Vacuum Config", func(t *testing.T) {
+		// Simulate form data
+		formData := url.Values{
+			// "enabled" field omitted to simulate unchecked checkbox
+			"scan_interval_seconds_value":  {"4"},     // interval field
+			"scan_interval_seconds_unit":   {"hours"}, // interval unit
+			"max_concurrent":               {"3"},     // number field
+			"garbage_threshold":            {"0.4"},   // float field
+			"min_volume_age_seconds_value": {"2"},     // interval field
+			"min_volume_age_seconds_unit":  {"days"},  // interval unit
+			"min_interval_seconds_value":   {"1"},     // interval field
+			"min_interval_seconds_unit":    {"days"},  // interval unit
+		}
+
+		// Get schema
+		schema := tasks.GetTaskConfigSchema("vacuum")
+		if schema == nil {
+			t.Fatal("Failed to get vacuum schema")
+		}
+
+		// Create config instance
+		config := &vacuum.Config{}
+
+		// Parse form data
+		err := h.parseTaskConfigFromForm(formData, schema, config)
+		if err != nil {
+			t.Fatalf("Failed to parse form data: %v", err)
+		}
+
+		// Verify embedded struct fields were set correctly
+		if config.Enabled {
+			t.Errorf("Expected Enabled=false, got %v", config.Enabled)
+		}
+
+		if config.ScanIntervalSeconds != 14400 { // 4 hours * 3600
+			t.Errorf("Expected ScanIntervalSeconds=14400, got %v", config.ScanIntervalSeconds)
+		}
+
+		if config.MaxConcurrent != 3 {
+			t.Errorf("Expected MaxConcurrent=3, got %v", config.MaxConcurrent)
+		}
+
+		// Verify vacuum-specific fields were set correctly
+		if config.GarbageThreshold != 0.4 {
+			t.Errorf("Expected GarbageThreshold=0.4, got %v", config.GarbageThreshold)
+		}
+
+		if config.MinVolumeAgeSeconds != 172800 { // 2 days * 86400
+			t.Errorf("Expected MinVolumeAgeSeconds=172800, got %v", config.MinVolumeAgeSeconds)
+		}
+
+		if config.MinIntervalSeconds != 86400 { // 1 day * 86400
+			t.Errorf("Expected MinIntervalSeconds=86400, got %v", config.MinIntervalSeconds)
+		}
+	})
+
+	// Test with erasure coding config
+	t.Run("Erasure Coding Config", func(t *testing.T) {
+		// Simulate form data
+		formData := url.Values{
+			"enabled":                     {"on"},              // checkbox field
+			"scan_interval_seconds_value": {"2"},               // interval field
+			"scan_interval_seconds_unit":  {"hours"},           // interval unit
+			"max_concurrent":              {"1"},               // number field
+			"quiet_for_seconds_value":     {"10"},              // interval field
+			"quiet_for_seconds_unit":      {"minutes"},         // interval unit
+			"fullness_ratio":              {"0.85"},            // float field
+			"collection_filter":           {"test_collection"}, // string field
+			"min_size_mb":                 {"50"},              // number field
+		}
+
+		// Get schema
+		schema := tasks.GetTaskConfigSchema("erasure_coding")
+		if schema == nil {
+			t.Fatal("Failed to get erasure_coding schema")
+		}
+
+		// Create config instance
+		config := &erasure_coding.Config{}
+
+		// Parse form data
+		err := h.parseTaskConfigFromForm(formData, schema, config)
+		if err != nil {
+			t.Fatalf("Failed to parse form data: %v", err)
+		}
+
+		// Verify embedded struct fields were set correctly
+		if !config.Enabled {
+			t.Errorf("Expected Enabled=true, got %v", config.Enabled)
+		}
+
+		if config.ScanIntervalSeconds != 7200 { // 2 hours * 3600
+			t.Errorf("Expected ScanIntervalSeconds=7200, got %v", config.ScanIntervalSeconds)
+		}
+
+		if config.MaxConcurrent != 1 {
+			t.Errorf("Expected MaxConcurrent=1, got %v", config.MaxConcurrent)
+		}
+
+		// Verify erasure coding-specific fields were set correctly
+		if config.QuietForSeconds != 600 { // 10 minutes * 60
+			t.Errorf("Expected QuietForSeconds=600, got %v", config.QuietForSeconds)
+		}
+
+		if config.FullnessRatio != 0.85 {
+			t.Errorf("Expected FullnessRatio=0.85, got %v", config.FullnessRatio)
+		}
+
+		if config.CollectionFilter != "test_collection" {
+			t.Errorf("Expected CollectionFilter='test_collection', got %v", config.CollectionFilter)
+		}
+
+		if config.MinSizeMB != 50 {
+			t.Errorf("Expected MinSizeMB=50, got %v", config.MinSizeMB)
+		}
+	})
+}
+
+func TestConfigurationValidation(t *testing.T) {
+	// Test that config structs can be validated and converted to protobuf format
+	taskTypes := []struct {
+		name   string
+		config interface{}
+	}{
+		{
+			"balance",
+			&balance.Config{
+				BaseConfig: base.BaseConfig{
+					Enabled:             true,
+					ScanIntervalSeconds: 2400,
+					MaxConcurrent:       3,
+				},
+				ImbalanceThreshold: 0.18,
+				MinServerCount:     4,
+			},
+		},
+		{
+			"vacuum",
+			&vacuum.Config{
+				BaseConfig: base.BaseConfig{
+					Enabled:             false,
+					ScanIntervalSeconds: 7200,
+					MaxConcurrent:       2,
+				},
+				GarbageThreshold:    0.35,
+				MinVolumeAgeSeconds: 86400,
+				MinIntervalSeconds:  604800,
+			},
+		},
+		{
+			"erasure_coding",
+			&erasure_coding.Config{
+				BaseConfig: base.BaseConfig{
+					Enabled:             true,
+					ScanIntervalSeconds: 3600,
+					MaxConcurrent:       1,
+				},
+				QuietForSeconds:  900,
+				FullnessRatio:    0.9,
+				CollectionFilter: "important",
+				MinSizeMB:        100,
+			},
+		},
+	}
+
+	for _, test := range taskTypes {
+		t.Run(test.name, func(t *testing.T) {
+			// Test that configs can be converted to protobuf TaskPolicy
+			switch cfg := test.config.(type) {
+			case *balance.Config:
+				policy := cfg.ToTaskPolicy()
+				if policy == nil {
+					t.Fatal("ToTaskPolicy returned nil")
+				}
+				if policy.Enabled != cfg.Enabled {
+					t.Errorf("Expected Enabled=%v, got %v", cfg.Enabled, policy.Enabled)
+				}
+				if policy.MaxConcurrent != int32(cfg.MaxConcurrent) {
+					t.Errorf("Expected MaxConcurrent=%v, got %v", cfg.MaxConcurrent, policy.MaxConcurrent)
+				}
+			case *vacuum.Config:
+				policy := cfg.ToTaskPolicy()
+				if policy == nil {
+					t.Fatal("ToTaskPolicy returned nil")
+				}
+				if policy.Enabled != cfg.Enabled {
+					t.Errorf("Expected Enabled=%v, got %v", cfg.Enabled, policy.Enabled)
+				}
+				if policy.MaxConcurrent != int32(cfg.MaxConcurrent) {
+					t.Errorf("Expected MaxConcurrent=%v, got %v", cfg.MaxConcurrent, policy.MaxConcurrent)
+				}
+			case *erasure_coding.Config:
+				policy := cfg.ToTaskPolicy()
+				if policy == nil {
+					t.Fatal("ToTaskPolicy returned nil")
+				}
+				if policy.Enabled != cfg.Enabled {
+					t.Errorf("Expected Enabled=%v, got %v", cfg.Enabled, policy.Enabled)
+				}
+				if policy.MaxConcurrent != int32(cfg.MaxConcurrent) {
+					t.Errorf("Expected MaxConcurrent=%v, got %v", cfg.MaxConcurrent, policy.MaxConcurrent)
+				}
+			default:
+				t.Fatalf("Unknown config type: %T", test.config)
+			}
+
+			// Test that configs can be validated
+			switch cfg := test.config.(type) {
+			case *balance.Config:
+				if err := cfg.Validate(); err != nil {
+					t.Errorf("Validation failed: %v", err)
+				}
+			case *vacuum.Config:
+				if err := cfg.Validate(); err != nil {
+					t.Errorf("Validation failed: %v", err)
+				}
+			case *erasure_coding.Config:
+				if err := cfg.Validate(); err != nil {
+					t.Errorf("Validation failed: %v", err)
+				}
+			}
+		})
+	}
+}
+
+func TestParseFieldFromForm_EdgeCases(t *testing.T) {
+	h := &MaintenanceHandlers{}
+
+	// Test checkbox parsing (boolean fields)
+	t.Run("Checkbox Fields", func(t *testing.T) {
+		tests := []struct {
+			name          string
+			formData      url.Values
+			expectedValue bool
+		}{
+			{"Checked checkbox", url.Values{"test_field": {"on"}}, true},
+			{"Unchecked checkbox", url.Values{}, false},
+			{"Empty value checkbox", url.Values{"test_field": {""}}, true}, // Present but empty means checked
+		}
+
+		for _, test := range tests {
+			t.Run(test.name, func(t *testing.T) {
+				schema := &tasks.TaskConfigSchema{
+					Schema: config.Schema{
+						Fields: []*config.Field{
+							{
+								JSONName:  "test_field",
+								Type:      config.FieldTypeBool,
+								InputType: "checkbox",
+							},
+						},
+					},
+				}
+
+				type TestConfig struct {
+					TestField bool `json:"test_field"`
+				}
+
+				config := &TestConfig{}
+				err := h.parseTaskConfigFromForm(test.formData, schema, config)
+				if err != nil {
+					t.Fatalf("parseTaskConfigFromForm failed: %v", err)
+				}
+
+				if config.TestField != test.expectedValue {
+					t.Errorf("Expected %v, got %v", test.expectedValue, config.TestField)
+				}
+			})
+		}
+	})
+
+	// Test interval parsing
+	t.Run("Interval Fields", func(t *testing.T) {
+		tests := []struct {
+			name         string
+			value        string
+			unit         string
+			expectedSecs int
+		}{
+			{"Minutes", "30", "minutes", 1800},
+			{"Hours", "2", "hours", 7200},
+			{"Days", "1", "days", 86400},
+		}
+
+		for _, test := range tests {
+			t.Run(test.name, func(t *testing.T) {
+				formData := url.Values{
+					"test_field_value": {test.value},
+					"test_field_unit":  {test.unit},
+				}
+
+				schema := &tasks.TaskConfigSchema{
+					Schema: config.Schema{
+						Fields: []*config.Field{
+							{
+								JSONName:  "test_field",
+								Type:      config.FieldTypeInterval,
+								InputType: "interval",
+							},
+						},
+					},
+				}
+
+				type TestConfig struct {
+					TestField int `json:"test_field"`
+				}
+
+				config := &TestConfig{}
+				err := h.parseTaskConfigFromForm(formData, schema, config)
+				if err != nil {
+					t.Fatalf("parseTaskConfigFromForm failed: %v", err)
+				}
+
+				if config.TestField != test.expectedSecs {
+					t.Errorf("Expected %d seconds, got %d", test.expectedSecs, config.TestField)
+				}
+			})
+		}
+	})
+}

weed/admin/handlers/plugin_handlers.go

@@ -1,67 +0,0 @@
-package handlers
-
-import (
-	"bytes"
-	"net/http"
-
-	"github.com/gin-gonic/gin"
-	"github.com/seaweedfs/seaweedfs/weed/admin/dash"
-	"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
-	"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
-)
-
-// PluginHandlers handles plugin UI pages.
-type PluginHandlers struct {
-	adminServer *dash.AdminServer
-}
-
-// NewPluginHandlers creates a new instance of PluginHandlers.
-func NewPluginHandlers(adminServer *dash.AdminServer) *PluginHandlers {
-	return &PluginHandlers{
-		adminServer: adminServer,
-	}
-}
-
-// ShowPlugin displays plugin overview page.
-func (h *PluginHandlers) ShowPlugin(c *gin.Context) {
-	h.renderPluginPage(c, "overview")
-}
-
-// ShowPluginConfiguration displays plugin configuration page.
-func (h *PluginHandlers) ShowPluginConfiguration(c *gin.Context) {
-	h.renderPluginPage(c, "configuration")
-}
-
-// ShowPluginDetection displays plugin detection jobs page.
-func (h *PluginHandlers) ShowPluginDetection(c *gin.Context) {
-	h.renderPluginPage(c, "detection")
-}
-
-// ShowPluginQueue displays plugin job queue page.
-func (h *PluginHandlers) ShowPluginQueue(c *gin.Context) {
-	h.renderPluginPage(c, "queue")
-}
-
-// ShowPluginExecution displays plugin execution jobs page.
-func (h *PluginHandlers) ShowPluginExecution(c *gin.Context) {
-	h.renderPluginPage(c, "execution")
-}
-
-// ShowPluginMonitoring displays plugin monitoring page.
-func (h *PluginHandlers) ShowPluginMonitoring(c *gin.Context) {
-	// Backward-compatible alias for the old monitoring URL.
-	h.renderPluginPage(c, "detection")
-}
-
-func (h *PluginHandlers) renderPluginPage(c *gin.Context, page string) {
-	component := app.Plugin(page)
-	layoutComponent := layout.Layout(c, component)
-
-	var buf bytes.Buffer
-	if err := layoutComponent.Render(c.Request.Context(), &buf); err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
-		return
-	}
-
-	c.Data(http.StatusOK, "text/html; charset=utf-8", buf.Bytes())
-}

weed/admin/plugin/DESIGN.md

@@ -1,205 +0,0 @@
-# Admin Worker Plugin System (Design)
-
-This document describes the plugin system for admin-managed workers, implemented in parallel with the current maintenance/worker mechanism.
-
-## Scope
-
-- Add a new plugin protocol and runtime model for multi-language workers.
-- Keep all current admin + worker code paths untouched.
-- Use gRPC for all admin-worker communication.
-- Let workers describe job configuration UI declaratively via protobuf.
-- Persist all job type configuration under admin server data directory.
-- Support detector workers and executor workers per job type.
-- Add end-to-end workflow observability (activities, active jobs, progress).
-
-## New Contract
-
-- Proto file: `weed/pb/plugin.proto`
-- gRPC service: `PluginControlService.WorkerStream`
-- Connection model: worker-initiated long-lived bidirectional stream.
-
-Why this model:
-
-- Works for workers in any language with gRPC support.
-- Avoids admin dialing constraints in NAT/private networks.
-- Allows command/response, progress streaming, and heartbeat over one channel.
-
-## Core Runtime Components (Admin Side)
-
-1. `PluginRegistry`
-- Tracks connected workers and their per-job-type capabilities.
-- Maintains liveness via heartbeat timeout.
-
-2. `SchemaCoordinator`
-- For each job type, asks one capable worker for `JobTypeDescriptor`.
-- Caches descriptor version and refresh timestamp.
-
-3. `ConfigStore`
-- Persists descriptor + saved config values in `dataDir`.
-- Stores both:
-  - Admin-owned runtime config (detection interval, dispatch concurrency, retry).
-  - Worker-owned config values (plugin-specific detection/execution knobs).
-
-4. `DetectorScheduler`
-- Per job type, chooses one detector worker (`can_detect=true`).
-- Sends `RunDetectionRequest` with saved configs + cluster context.
-- Accepts `DetectionProposals`, dedupes by `dedupe_key`, inserts jobs.
-
-5. `JobDispatcher`
-- Chooses executor worker (`can_execute=true`) for each pending job.
-- Sends `ExecuteJobRequest`.
-- Consumes `JobProgressUpdate` and `JobCompleted`.
-
-6. `WorkflowMonitor`
-- Builds live counters and timeline from events:
-  - activities per job type,
-  - active jobs,
-  - per-job progress/state,
-  - worker health/load.
-
-## Worker Responsibilities
-
-1. Register capabilities on connect (`WorkerHello`).
-2. Expose job type descriptor (`ConfigSchemaResponse`) including UI schemas:
-- admin config form,
-- worker config form,
-- defaults.
-3. Run detection on demand (`RunDetectionRequest`) and return proposals.
-4. Execute assigned jobs (`ExecuteJobRequest`) and stream progress.
-5. Heartbeat regularly with slot usage and running work.
-6. Handle cancellation requests (`CancelRequest`) for in-flight detection/execution.
-
-## Declarative UI Model
-
-UI is fully derived from protobuf schema:
-
-- `ConfigForm`
-- `ConfigSection`
-- `ConfigField`
-- `ConfigOption`
-- `ValidationRule`
-- `ConfigValue` (typed scalar/list/map/object value container)
-
-Result:
-
-- Admin can render forms without hardcoded task structs.
-- New job types can ship UI schema from worker binary alone.
-- Worker language is irrelevant as long as it can emit protobuf messages.
-
-## Detection and Dispatch Flow
-
-1. Worker connects and registers capabilities.
-2. Admin requests descriptor per job type.
-3. Admin persists descriptor and editable config values.
-4. On detection interval (admin-owned setting):
-- Admin chooses one detector worker for that job type.
-- Sends `RunDetectionRequest` with:
-  - `AdminRuntimeConfig`,
-  - `admin_config_values`,
-  - `worker_config_values`,
-  - `ClusterContext` (master/filer/volume grpc locations, metadata).
-5. Detector emits `DetectionProposals` and `DetectionComplete`.
-6. Admin dedupes and enqueues jobs.
-7. Dispatcher assigns jobs to any eligible executor worker.
-8. Executor emits `JobProgressUpdate` and `JobCompleted`.
-9. Monitor updates workflow UI in near-real-time.
-
-## Persistence Layout (Admin Data Dir)
-
-Current layout under `<admin-data-dir>/plugin/`:
-
-- `job_types/<job_type>/descriptor.pb`
-- `job_types/<job_type>/descriptor.json`
-- `job_types/<job_type>/config.pb`
-- `job_types/<job_type>/config.json`
-- `job_types/<job_type>/runs.json`
-- `jobs/tracked_jobs.json`
-- `activities/activities.json`
-
-`config.pb` should use `PersistedJobTypeConfig` from `plugin.proto`.
-
-## Admin UI
-
-- Route: `/plugin`
-- Includes:
-  - runtime status,
-  - workers/capabilities,
-  - declarative descriptor-driven config forms,
-  - run history (last 10 success + last 10 errors),
-  - tracked jobs and activity stream,
-  - manual actions for schema refresh, detection, and detect+execute workflow.
-
-## Scheduling Policy (Initial)
-
-Detector selection per job type:
-- only workers with `can_detect=true`.
-- prefer healthy worker with highest free detection slots.
-- lease ends when heartbeat timeout or stream drop.
-
-Execution dispatch:
-- only workers with `can_execute=true`.
-- select by available execution slots and least active jobs.
-- retry on failure using admin runtime retry config.
-
-## Safety and Reliability
-
-- Idempotency: dedupe proposals by (`job_type`, `dedupe_key`).
-- Backpressure: enforce max jobs per detection run.
-- Timeouts: detection and execution timeout from admin runtime config.
-- Replay-safe persistence: write job state changes before emitting UI events.
-- Heartbeat-based failover for detector/executor reassignment.
-
-## Backward Compatibility
-
-- Legacy `worker.proto` runtime remains internally available where still referenced.
-- External CLI worker path is moved to plugin runtime behavior.
-- Runtime is enabled by default on admin worker gRPC server.
-
-## Incremental Rollout Plan
-
-Phase 1
-- Introduce protocol and storage models only.
-
-Phase 2
-- Build admin registry/scheduler/dispatcher behind feature flag.
-
-Phase 3
-- Add dedicated plugin UI pages and metrics.
-
-Phase 4
-- Port one existing job type (e.g. vacuum) as external worker plugin.
-
-Phase 4 status (starter)
-- Added `weed worker` command as an external `plugin.proto` worker process.
-- Initial handler implements `vacuum` job type with:
-  - declarative descriptor/config form response (`ConfigSchemaResponse`),
-  - detection via master topology scan (`RunDetectionRequest`),
-  - execution via existing vacuum task logic (`ExecuteJobRequest`),
-  - heartbeat/load reporting for monitor UI.
-- Legacy maintenance-worker-specific CLI path is removed.
-
-Run example:
-- Start admin: `weed admin -master=localhost:9333`
-- Start worker: `weed worker -admin=localhost:23646`
-- Optional explicit job type: `weed worker -admin=localhost:23646 -jobType=vacuum`
-- Optional stable worker ID persistence: `weed worker -admin=localhost:23646 -workingDir=/var/lib/seaweedfs-plugin`
-
-Phase 5
-- Migrate remaining job types and deprecate old mechanism.
-
-## Agreed Defaults
-
-1. Detector multiplicity
-- Exactly one detector worker per job type at a time. Admin selects one worker and runs detection there.
-
-2. Secret handling
-- No encryption at rest required for plugin config in this phase.
-
-3. Schema compatibility
-- No migration policy required yet; this is a new system.
-
-4. Execution ownership
-- Same worker is allowed to do both detection and execution.
-
-5. Retention
-- Keep last 10 successful runs and last 10 error runs per job type.

weed/admin/plugin/config_store.go

@@ -1,739 +0,0 @@
-package plugin
-
-import (
-	"encoding/json"
-	"fmt"
-	"net/url"
-	"os"
-	"path/filepath"
-	"regexp"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-	"google.golang.org/protobuf/encoding/protojson"
-	"google.golang.org/protobuf/proto"
-)
-
-const (
-	pluginDirName           = "plugin"
-	jobTypesDirName         = "job_types"
-	jobsDirName             = "jobs"
-	jobDetailsDirName       = "job_details"
-	activitiesDirName       = "activities"
-	descriptorPBFileName    = "descriptor.pb"
-	descriptorJSONFileName  = "descriptor.json"
-	configPBFileName        = "config.pb"
-	configJSONFileName      = "config.json"
-	runsJSONFileName        = "runs.json"
-	trackedJobsJSONFileName = "tracked_jobs.json"
-	activitiesJSONFileName  = "activities.json"
-	defaultDirPerm          = 0o755
-	defaultFilePerm         = 0o644
-)
-
-// validJobTypePattern is the canonical pattern for safe job type names.
-// Only letters, digits, underscore, dash, and dot are allowed, which prevents
-// path traversal because '/', '\\', and whitespace are rejected.
-var validJobTypePattern = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`)
-
-// ConfigStore persists plugin configuration and bounded run history.
-// If admin data dir is empty, it transparently falls back to in-memory mode.
-type ConfigStore struct {
-	configured bool
-	baseDir    string
-
-	mu sync.RWMutex
-
-	memDescriptors map[string]*plugin_pb.JobTypeDescriptor
-	memConfigs     map[string]*plugin_pb.PersistedJobTypeConfig
-	memRunHistory  map[string]*JobTypeRunHistory
-	memTrackedJobs []TrackedJob
-	memActivities  []JobActivity
-	memJobDetails  map[string]TrackedJob
-}
-
-func NewConfigStore(adminDataDir string) (*ConfigStore, error) {
-	store := &ConfigStore{
-		configured:     adminDataDir != "",
-		memDescriptors: make(map[string]*plugin_pb.JobTypeDescriptor),
-		memConfigs:     make(map[string]*plugin_pb.PersistedJobTypeConfig),
-		memRunHistory:  make(map[string]*JobTypeRunHistory),
-		memJobDetails:  make(map[string]TrackedJob),
-	}
-
-	if adminDataDir == "" {
-		return store, nil
-	}
-
-	store.baseDir = filepath.Join(adminDataDir, pluginDirName)
-	if err := os.MkdirAll(filepath.Join(store.baseDir, jobTypesDirName), defaultDirPerm); err != nil {
-		return nil, fmt.Errorf("create plugin job_types dir: %w", err)
-	}
-	if err := os.MkdirAll(filepath.Join(store.baseDir, jobsDirName), defaultDirPerm); err != nil {
-		return nil, fmt.Errorf("create plugin jobs dir: %w", err)
-	}
-	if err := os.MkdirAll(filepath.Join(store.baseDir, jobsDirName, jobDetailsDirName), defaultDirPerm); err != nil {
-		return nil, fmt.Errorf("create plugin job_details dir: %w", err)
-	}
-	if err := os.MkdirAll(filepath.Join(store.baseDir, activitiesDirName), defaultDirPerm); err != nil {
-		return nil, fmt.Errorf("create plugin activities dir: %w", err)
-	}
-
-	return store, nil
-}
-
-func (s *ConfigStore) IsConfigured() bool {
-	return s.configured
-}
-
-func (s *ConfigStore) BaseDir() string {
-	return s.baseDir
-}
-
-func (s *ConfigStore) SaveDescriptor(jobType string, descriptor *plugin_pb.JobTypeDescriptor) error {
-	if descriptor == nil {
-		return fmt.Errorf("descriptor is nil")
-	}
-	if _, err := sanitizeJobType(jobType); err != nil {
-		return err
-	}
-
-	clone := proto.Clone(descriptor).(*plugin_pb.JobTypeDescriptor)
-	if clone.JobType == "" {
-		clone.JobType = jobType
-	}
-
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	if !s.configured {
-		s.memDescriptors[jobType] = clone
-		return nil
-	}
-
-	jobTypeDir, err := s.ensureJobTypeDir(jobType)
-	if err != nil {
-		return err
-	}
-
-	pbPath := filepath.Join(jobTypeDir, descriptorPBFileName)
-	jsonPath := filepath.Join(jobTypeDir, descriptorJSONFileName)
-
-	if err := writeProtoFiles(clone, pbPath, jsonPath); err != nil {
-		return fmt.Errorf("save descriptor for %s: %w", jobType, err)
-	}
-
-	return nil
-}
-
-func (s *ConfigStore) LoadDescriptor(jobType string) (*plugin_pb.JobTypeDescriptor, error) {
-	if _, err := sanitizeJobType(jobType); err != nil {
-		return nil, err
-	}
-
-	s.mu.RLock()
-	if !s.configured {
-		d := s.memDescriptors[jobType]
-		s.mu.RUnlock()
-		if d == nil {
-			return nil, nil
-		}
-		return proto.Clone(d).(*plugin_pb.JobTypeDescriptor), nil
-	}
-	s.mu.RUnlock()
-
-	pbPath := filepath.Join(s.baseDir, jobTypesDirName, jobType, descriptorPBFileName)
-	data, err := os.ReadFile(pbPath)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
-		}
-		return nil, fmt.Errorf("read descriptor for %s: %w", jobType, err)
-	}
-
-	var descriptor plugin_pb.JobTypeDescriptor
-	if err := proto.Unmarshal(data, &descriptor); err != nil {
-		return nil, fmt.Errorf("unmarshal descriptor for %s: %w", jobType, err)
-	}
-	return &descriptor, nil
-}
-
-func (s *ConfigStore) SaveJobTypeConfig(config *plugin_pb.PersistedJobTypeConfig) error {
-	if config == nil {
-		return fmt.Errorf("job type config is nil")
-	}
-	if config.JobType == "" {
-		return fmt.Errorf("job type config has empty job_type")
-	}
-	sanitizedJobType, err := sanitizeJobType(config.JobType)
-	if err != nil {
-		return err
-	}
-	// Use the sanitized job type going forward to ensure it is safe for filesystem paths.
-	config.JobType = sanitizedJobType
-
-	clone := proto.Clone(config).(*plugin_pb.PersistedJobTypeConfig)
-
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	if !s.configured {
-		s.memConfigs[config.JobType] = clone
-		return nil
-	}
-
-	jobTypeDir, err := s.ensureJobTypeDir(config.JobType)
-	if err != nil {
-		return err
-	}
-
-	pbPath := filepath.Join(jobTypeDir, configPBFileName)
-	jsonPath := filepath.Join(jobTypeDir, configJSONFileName)
-
-	if err := writeProtoFiles(clone, pbPath, jsonPath); err != nil {
-		return fmt.Errorf("save job type config for %s: %w", config.JobType, err)
-	}
-
-	return nil
-}
-
-func (s *ConfigStore) LoadJobTypeConfig(jobType string) (*plugin_pb.PersistedJobTypeConfig, error) {
-	if _, err := sanitizeJobType(jobType); err != nil {
-		return nil, err
-	}
-
-	s.mu.RLock()
-	if !s.configured {
-		cfg := s.memConfigs[jobType]
-		s.mu.RUnlock()
-		if cfg == nil {
-			return nil, nil
-		}
-		return proto.Clone(cfg).(*plugin_pb.PersistedJobTypeConfig), nil
-	}
-	s.mu.RUnlock()
-
-	pbPath := filepath.Join(s.baseDir, jobTypesDirName, jobType, configPBFileName)
-	data, err := os.ReadFile(pbPath)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
-		}
-		return nil, fmt.Errorf("read job type config for %s: %w", jobType, err)
-	}
-
-	var config plugin_pb.PersistedJobTypeConfig
-	if err := proto.Unmarshal(data, &config); err != nil {
-		return nil, fmt.Errorf("unmarshal job type config for %s: %w", jobType, err)
-	}
-
-	return &config, nil
-}
-
-func (s *ConfigStore) AppendRunRecord(jobType string, record *JobRunRecord) error {
-	if record == nil {
-		return fmt.Errorf("run record is nil")
-	}
-	if _, err := sanitizeJobType(jobType); err != nil {
-		return err
-	}
-
-	safeRecord := *record
-	if safeRecord.JobType == "" {
-		safeRecord.JobType = jobType
-	}
-	if safeRecord.CompletedAt == nil || safeRecord.CompletedAt.IsZero() {
-		safeRecord.CompletedAt = timeToPtr(time.Now().UTC())
-	}
-
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	history, err := s.loadRunHistoryLocked(jobType)
-	if err != nil {
-		return err
-	}
-
-	if safeRecord.Outcome == RunOutcomeSuccess {
-		history.SuccessfulRuns = append(history.SuccessfulRuns, safeRecord)
-	} else {
-		safeRecord.Outcome = RunOutcomeError
-		history.ErrorRuns = append(history.ErrorRuns, safeRecord)
-	}
-
-	history.SuccessfulRuns = trimRuns(history.SuccessfulRuns, MaxSuccessfulRunHistory)
-	history.ErrorRuns = trimRuns(history.ErrorRuns, MaxErrorRunHistory)
-	history.LastUpdatedTime = timeToPtr(time.Now().UTC())
-
-	return s.saveRunHistoryLocked(jobType, history)
-}
-
-func (s *ConfigStore) LoadRunHistory(jobType string) (*JobTypeRunHistory, error) {
-	if _, err := sanitizeJobType(jobType); err != nil {
-		return nil, err
-	}
-
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	history, err := s.loadRunHistoryLocked(jobType)
-	if err != nil {
-		return nil, err
-	}
-	return cloneRunHistory(history), nil
-}
-
-func (s *ConfigStore) SaveTrackedJobs(jobs []TrackedJob) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	clone := cloneTrackedJobs(jobs)
-
-	if !s.configured {
-		s.memTrackedJobs = clone
-		return nil
-	}
-
-	encoded, err := json.MarshalIndent(clone, "", "  ")
-	if err != nil {
-		return fmt.Errorf("encode tracked jobs: %w", err)
-	}
-
-	path := filepath.Join(s.baseDir, jobsDirName, trackedJobsJSONFileName)
-	if err := atomicWriteFile(path, encoded, defaultFilePerm); err != nil {
-		return fmt.Errorf("write tracked jobs: %w", err)
-	}
-	return nil
-}
-
-func (s *ConfigStore) LoadTrackedJobs() ([]TrackedJob, error) {
-	s.mu.RLock()
-	if !s.configured {
-		out := cloneTrackedJobs(s.memTrackedJobs)
-		s.mu.RUnlock()
-		return out, nil
-	}
-	s.mu.RUnlock()
-
-	path := filepath.Join(s.baseDir, jobsDirName, trackedJobsJSONFileName)
-	data, err := os.ReadFile(path)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
-		}
-		return nil, fmt.Errorf("read tracked jobs: %w", err)
-	}
-
-	var jobs []TrackedJob
-	if err := json.Unmarshal(data, &jobs); err != nil {
-		return nil, fmt.Errorf("parse tracked jobs: %w", err)
-	}
-	return cloneTrackedJobs(jobs), nil
-}
-
-func (s *ConfigStore) SaveJobDetail(job TrackedJob) error {
-	jobID, err := sanitizeJobID(job.JobID)
-	if err != nil {
-		return err
-	}
-
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	clone := cloneTrackedJob(job)
-	clone.JobID = jobID
-
-	if !s.configured {
-		s.memJobDetails[jobID] = clone
-		return nil
-	}
-
-	encoded, err := json.MarshalIndent(clone, "", "  ")
-	if err != nil {
-		return fmt.Errorf("encode job detail: %w", err)
-	}
-
-	path := filepath.Join(s.baseDir, jobsDirName, jobDetailsDirName, jobDetailFileName(jobID))
-	if err := atomicWriteFile(path, encoded, defaultFilePerm); err != nil {
-		return fmt.Errorf("write job detail: %w", err)
-	}
-
-	return nil
-}
-
-func (s *ConfigStore) LoadJobDetail(jobID string) (*TrackedJob, error) {
-	jobID, err := sanitizeJobID(jobID)
-	if err != nil {
-		return nil, err
-	}
-
-	s.mu.RLock()
-	if !s.configured {
-		job, ok := s.memJobDetails[jobID]
-		s.mu.RUnlock()
-		if !ok {
-			return nil, nil
-		}
-		clone := cloneTrackedJob(job)
-		return &clone, nil
-	}
-	s.mu.RUnlock()
-
-	path := filepath.Join(s.baseDir, jobsDirName, jobDetailsDirName, jobDetailFileName(jobID))
-	data, err := os.ReadFile(path)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
-		}
-		return nil, fmt.Errorf("read job detail: %w", err)
-	}
-
-	var job TrackedJob
-	if err := json.Unmarshal(data, &job); err != nil {
-		return nil, fmt.Errorf("parse job detail: %w", err)
-	}
-	clone := cloneTrackedJob(job)
-	return &clone, nil
-}
-
-func (s *ConfigStore) SaveActivities(activities []JobActivity) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	clone := cloneActivities(activities)
-
-	if !s.configured {
-		s.memActivities = clone
-		return nil
-	}
-
-	encoded, err := json.MarshalIndent(clone, "", "  ")
-	if err != nil {
-		return fmt.Errorf("encode activities: %w", err)
-	}
-
-	path := filepath.Join(s.baseDir, activitiesDirName, activitiesJSONFileName)
-	if err := atomicWriteFile(path, encoded, defaultFilePerm); err != nil {
-		return fmt.Errorf("write activities: %w", err)
-	}
-	return nil
-}
-
-func (s *ConfigStore) LoadActivities() ([]JobActivity, error) {
-	s.mu.RLock()
-	if !s.configured {
-		out := cloneActivities(s.memActivities)
-		s.mu.RUnlock()
-		return out, nil
-	}
-	s.mu.RUnlock()
-
-	path := filepath.Join(s.baseDir, activitiesDirName, activitiesJSONFileName)
-	data, err := os.ReadFile(path)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
-		}
-		return nil, fmt.Errorf("read activities: %w", err)
-	}
-
-	var activities []JobActivity
-	if err := json.Unmarshal(data, &activities); err != nil {
-		return nil, fmt.Errorf("parse activities: %w", err)
-	}
-	return cloneActivities(activities), nil
-}
-
-func (s *ConfigStore) ListJobTypes() ([]string, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-
-	jobTypeSet := make(map[string]struct{})
-
-	if !s.configured {
-		for jobType := range s.memDescriptors {
-			jobTypeSet[jobType] = struct{}{}
-		}
-		for jobType := range s.memConfigs {
-			jobTypeSet[jobType] = struct{}{}
-		}
-		for jobType := range s.memRunHistory {
-			jobTypeSet[jobType] = struct{}{}
-		}
-	} else {
-		jobTypesPath := filepath.Join(s.baseDir, jobTypesDirName)
-		entries, err := os.ReadDir(jobTypesPath)
-		if err != nil {
-			if os.IsNotExist(err) {
-				return []string{}, nil
-			}
-			return nil, fmt.Errorf("list job types: %w", err)
-		}
-		for _, entry := range entries {
-			if !entry.IsDir() {
-				continue
-			}
-			jobType := strings.TrimSpace(entry.Name())
-			if _, err := sanitizeJobType(jobType); err != nil {
-				continue
-			}
-			jobTypeSet[jobType] = struct{}{}
-		}
-	}
-
-	jobTypes := make([]string, 0, len(jobTypeSet))
-	for jobType := range jobTypeSet {
-		jobTypes = append(jobTypes, jobType)
-	}
-	sort.Strings(jobTypes)
-	return jobTypes, nil
-}
-
-func (s *ConfigStore) loadRunHistoryLocked(jobType string) (*JobTypeRunHistory, error) {
-	if !s.configured {
-		history, ok := s.memRunHistory[jobType]
-		if !ok {
-			history = &JobTypeRunHistory{JobType: jobType}
-			s.memRunHistory[jobType] = history
-		}
-		return cloneRunHistory(history), nil
-	}
-
-	runsPath := filepath.Join(s.baseDir, jobTypesDirName, jobType, runsJSONFileName)
-	data, err := os.ReadFile(runsPath)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return &JobTypeRunHistory{JobType: jobType}, nil
-		}
-		return nil, fmt.Errorf("read run history for %s: %w", jobType, err)
-	}
-
-	var history JobTypeRunHistory
-	if err := json.Unmarshal(data, &history); err != nil {
-		return nil, fmt.Errorf("parse run history for %s: %w", jobType, err)
-	}
-	if history.JobType == "" {
-		history.JobType = jobType
-	}
-	return &history, nil
-}
-
-func (s *ConfigStore) saveRunHistoryLocked(jobType string, history *JobTypeRunHistory) error {
-	if !s.configured {
-		s.memRunHistory[jobType] = cloneRunHistory(history)
-		return nil
-	}
-
-	jobTypeDir, err := s.ensureJobTypeDir(jobType)
-	if err != nil {
-		return err
-	}
-
-	encoded, err := json.MarshalIndent(history, "", "  ")
-	if err != nil {
-		return fmt.Errorf("encode run history for %s: %w", jobType, err)
-	}
-
-	runsPath := filepath.Join(jobTypeDir, runsJSONFileName)
-	if err := atomicWriteFile(runsPath, encoded, defaultFilePerm); err != nil {
-		return fmt.Errorf("write run history for %s: %w", jobType, err)
-	}
-	return nil
-}
-
-func (s *ConfigStore) ensureJobTypeDir(jobType string) (string, error) {
-	if !s.configured {
-		return "", nil
-	}
-	jobTypeDir := filepath.Join(s.baseDir, jobTypesDirName, jobType)
-	if err := os.MkdirAll(jobTypeDir, defaultDirPerm); err != nil {
-		return "", fmt.Errorf("create job type dir for %s: %w", jobType, err)
-	}
-	return jobTypeDir, nil
-}
-
-func sanitizeJobType(jobType string) (string, error) {
-	jobType = strings.TrimSpace(jobType)
-	if jobType == "" {
-		return "", fmt.Errorf("job type is empty")
-	}
-	// Enforce a strict, path-safe pattern for job types: only letters, digits, underscore, dash and dot.
-	// This prevents path traversal because '/', '\\' and whitespace are rejected.
-	if !validJobTypePattern.MatchString(jobType) {
-		return "", fmt.Errorf("invalid job type %q: must match %s", jobType, validJobTypePattern.String())
-	}
-	return jobType, nil
-}
-
-// validJobIDPattern allows letters, digits, dash, underscore, and dot.
-// url.PathEscape in jobDetailFileName provides a second layer of defense.
-var validJobIDPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`)
-
-func sanitizeJobID(jobID string) (string, error) {
-	jobID = strings.TrimSpace(jobID)
-	if jobID == "" {
-		return "", fmt.Errorf("job id is empty")
-	}
-	if !validJobIDPattern.MatchString(jobID) {
-		return "", fmt.Errorf("invalid job id %q: must match %s", jobID, validJobIDPattern.String())
-	}
-	return jobID, nil
-}
-
-func jobDetailFileName(jobID string) string {
-	return url.PathEscape(jobID) + ".json"
-}
-
-func trimRuns(runs []JobRunRecord, maxKeep int) []JobRunRecord {
-	if len(runs) == 0 {
-		return runs
-	}
-	sort.Slice(runs, func(i, j int) bool {
-		ti := time.Time{}
-		if runs[i].CompletedAt != nil {
-			ti = *runs[i].CompletedAt
-		}
-		tj := time.Time{}
-		if runs[j].CompletedAt != nil {
-			tj = *runs[j].CompletedAt
-		}
-		return ti.After(tj)
-	})
-	if len(runs) > maxKeep {
-		runs = runs[:maxKeep]
-	}
-	return runs
-}
-
-func cloneRunHistory(in *JobTypeRunHistory) *JobTypeRunHistory {
-	if in == nil {
-		return nil
-	}
-	out := *in
-	if in.SuccessfulRuns != nil {
-		out.SuccessfulRuns = append([]JobRunRecord(nil), in.SuccessfulRuns...)
-	}
-	if in.ErrorRuns != nil {
-		out.ErrorRuns = append([]JobRunRecord(nil), in.ErrorRuns...)
-	}
-	return &out
-}
-
-func cloneTrackedJobs(in []TrackedJob) []TrackedJob {
-	if len(in) == 0 {
-		return nil
-	}
-
-	out := make([]TrackedJob, len(in))
-	for i := range in {
-		out[i] = cloneTrackedJob(in[i])
-	}
-	return out
-}
-
-func cloneTrackedJob(in TrackedJob) TrackedJob {
-	out := in
-	if in.Parameters != nil {
-		out.Parameters = make(map[string]interface{}, len(in.Parameters))
-		for key, value := range in.Parameters {
-			out.Parameters[key] = deepCopyGenericValue(value)
-		}
-	}
-	if in.Labels != nil {
-		out.Labels = make(map[string]string, len(in.Labels))
-		for key, value := range in.Labels {
-			out.Labels[key] = value
-		}
-	}
-	if in.ResultOutputValues != nil {
-		out.ResultOutputValues = make(map[string]interface{}, len(in.ResultOutputValues))
-		for key, value := range in.ResultOutputValues {
-			out.ResultOutputValues[key] = deepCopyGenericValue(value)
-		}
-	}
-	return out
-}
-
-func deepCopyGenericValue(val interface{}) interface{} {
-	switch v := val.(type) {
-	case map[string]interface{}:
-		res := make(map[string]interface{}, len(v))
-		for k, val := range v {
-			res[k] = deepCopyGenericValue(val)
-		}
-		return res
-	case []interface{}:
-		res := make([]interface{}, len(v))
-		for i, val := range v {
-			res[i] = deepCopyGenericValue(val)
-		}
-		return res
-	default:
-		return v
-	}
-}
-
-func cloneActivities(in []JobActivity) []JobActivity {
-	if len(in) == 0 {
-		return nil
-	}
-
-	out := make([]JobActivity, len(in))
-	for i := range in {
-		out[i] = in[i]
-		if in[i].Details != nil {
-			out[i].Details = make(map[string]interface{}, len(in[i].Details))
-			for key, value := range in[i].Details {
-				out[i].Details[key] = deepCopyGenericValue(value)
-			}
-		}
-	}
-	return out
-}
-
-// writeProtoFiles writes message to both a binary protobuf file (pbPath) and a
-// human-readable JSON file (jsonPath) using atomicWriteFile for each.
-// The .pb file is the authoritative source of truth: all reads use proto.Unmarshal
-// on the .pb file. The .json file is for human inspection only, so a partial
-// failure where .pb succeeds but .json fails leaves the store in a consistent state.
-func writeProtoFiles(message proto.Message, pbPath string, jsonPath string) error {
-	pbData, err := proto.Marshal(message)
-	if err != nil {
-		return fmt.Errorf("marshal protobuf: %w", err)
-	}
-	if err := atomicWriteFile(pbPath, pbData, defaultFilePerm); err != nil {
-		return fmt.Errorf("write protobuf file: %w", err)
-	}
-
-	jsonData, err := protojson.MarshalOptions{
-		Multiline:       true,
-		Indent:          "  ",
-		EmitUnpopulated: true,
-	}.Marshal(message)
-	if err != nil {
-		return fmt.Errorf("marshal json: %w", err)
-	}
-	if err := atomicWriteFile(jsonPath, jsonData, defaultFilePerm); err != nil {
-		return fmt.Errorf("write json file: %w", err)
-	}
-
-	return nil
-}
-func atomicWriteFile(filename string, data []byte, perm os.FileMode) error {
-	dir := filepath.Dir(filename)
-	if err := os.MkdirAll(dir, defaultDirPerm); err != nil {
-		return fmt.Errorf("create directory %s: %w", dir, err)
-	}
-	tmpFile := filename + ".tmp"
-	if err := os.WriteFile(tmpFile, data, perm); err != nil {
-		return err
-	}
-	if err := os.Rename(tmpFile, filename); err != nil {
-		_ = os.Remove(tmpFile)
-		return err
-	}
-	return nil
-}

weed/admin/plugin/config_store_test.go

@@ -1,257 +0,0 @@
-package plugin
-
-import (
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-)
-
-func TestConfigStoreDescriptorRoundTrip(t *testing.T) {
-	t.Parallel()
-
-	tempDir := t.TempDir()
-	store, err := NewConfigStore(tempDir)
-	if err != nil {
-		t.Fatalf("NewConfigStore: %v", err)
-	}
-
-	descriptor := &plugin_pb.JobTypeDescriptor{
-		JobType:           "vacuum",
-		DisplayName:       "Vacuum",
-		Description:       "Vacuum volumes",
-		DescriptorVersion: 1,
-	}
-
-	if err := store.SaveDescriptor("vacuum", descriptor); err != nil {
-		t.Fatalf("SaveDescriptor: %v", err)
-	}
-
-	got, err := store.LoadDescriptor("vacuum")
-	if err != nil {
-		t.Fatalf("LoadDescriptor: %v", err)
-	}
-	if got == nil {
-		t.Fatalf("LoadDescriptor: nil descriptor")
-	}
-	if got.DisplayName != descriptor.DisplayName {
-		t.Fatalf("unexpected display name: got %q want %q", got.DisplayName, descriptor.DisplayName)
-	}
-
-}
-
-func TestConfigStoreRunHistoryRetention(t *testing.T) {
-	t.Parallel()
-
-	store, err := NewConfigStore(t.TempDir())
-	if err != nil {
-		t.Fatalf("NewConfigStore: %v", err)
-	}
-
-	base := time.Now().UTC().Add(-24 * time.Hour)
-	for i := 0; i < 15; i++ {
-		err := store.AppendRunRecord("balance", &JobRunRecord{
-			RunID:       "s" + time.Duration(i).String(),
-			JobID:       "job-success",
-			JobType:     "balance",
-			WorkerID:    "worker-a",
-			Outcome:     RunOutcomeSuccess,
-			CompletedAt: timeToPtr(base.Add(time.Duration(i) * time.Minute)),
-		})
-		if err != nil {
-			t.Fatalf("AppendRunRecord success[%d]: %v", i, err)
-		}
-	}
-
-	for i := 0; i < 12; i++ {
-		err := store.AppendRunRecord("balance", &JobRunRecord{
-			RunID:       "e" + time.Duration(i).String(),
-			JobID:       "job-error",
-			JobType:     "balance",
-			WorkerID:    "worker-b",
-			Outcome:     RunOutcomeError,
-			CompletedAt: timeToPtr(base.Add(time.Duration(i) * time.Minute)),
-		})
-		if err != nil {
-			t.Fatalf("AppendRunRecord error[%d]: %v", i, err)
-		}
-	}
-
-	history, err := store.LoadRunHistory("balance")
-	if err != nil {
-		t.Fatalf("LoadRunHistory: %v", err)
-	}
-	if len(history.SuccessfulRuns) != MaxSuccessfulRunHistory {
-		t.Fatalf("successful retention mismatch: got %d want %d", len(history.SuccessfulRuns), MaxSuccessfulRunHistory)
-	}
-	if len(history.ErrorRuns) != MaxErrorRunHistory {
-		t.Fatalf("error retention mismatch: got %d want %d", len(history.ErrorRuns), MaxErrorRunHistory)
-	}
-
-	for i := 1; i < len(history.SuccessfulRuns); i++ {
-		t1 := time.Time{}
-		if history.SuccessfulRuns[i-1].CompletedAt != nil {
-			t1 = *history.SuccessfulRuns[i-1].CompletedAt
-		}
-		t2 := time.Time{}
-		if history.SuccessfulRuns[i].CompletedAt != nil {
-			t2 = *history.SuccessfulRuns[i].CompletedAt
-		}
-		if t1.Before(t2) {
-			t.Fatalf("successful run order not descending at %d", i)
-		}
-	}
-	for i := 1; i < len(history.ErrorRuns); i++ {
-		t1 := time.Time{}
-		if history.ErrorRuns[i-1].CompletedAt != nil {
-			t1 = *history.ErrorRuns[i-1].CompletedAt
-		}
-		t2 := time.Time{}
-		if history.ErrorRuns[i].CompletedAt != nil {
-			t2 = *history.ErrorRuns[i].CompletedAt
-		}
-		if t1.Before(t2) {
-			t.Fatalf("error run order not descending at %d", i)
-		}
-	}
-}
-
-func TestConfigStoreListJobTypes(t *testing.T) {
-	t.Parallel()
-
-	store, err := NewConfigStore("")
-	if err != nil {
-		t.Fatalf("NewConfigStore: %v", err)
-	}
-
-	if err := store.SaveDescriptor("vacuum", &plugin_pb.JobTypeDescriptor{JobType: "vacuum"}); err != nil {
-		t.Fatalf("SaveDescriptor: %v", err)
-	}
-	if err := store.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
-		JobType:      "balance",
-		AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true},
-	}); err != nil {
-		t.Fatalf("SaveJobTypeConfig: %v", err)
-	}
-	if err := store.AppendRunRecord("ec", &JobRunRecord{Outcome: RunOutcomeSuccess, CompletedAt: timeToPtr(time.Now().UTC())}); err != nil {
-		t.Fatalf("AppendRunRecord: %v", err)
-	}
-
-	got, err := store.ListJobTypes()
-	if err != nil {
-		t.Fatalf("ListJobTypes: %v", err)
-	}
-	want := []string{"balance", "ec", "vacuum"}
-	if !reflect.DeepEqual(got, want) {
-		t.Fatalf("unexpected job types: got=%v want=%v", got, want)
-	}
-}
-
-func TestConfigStoreMonitorStateRoundTrip(t *testing.T) {
-	t.Parallel()
-
-	store, err := NewConfigStore(t.TempDir())
-	if err != nil {
-		t.Fatalf("NewConfigStore: %v", err)
-	}
-
-	tracked := []TrackedJob{
-		{
-			JobID:     "job-1",
-			JobType:   "vacuum",
-			State:     "running",
-			Progress:  55,
-			WorkerID:  "worker-a",
-			CreatedAt: timeToPtr(time.Now().UTC().Add(-2 * time.Minute)),
-			UpdatedAt: timeToPtr(time.Now().UTC().Add(-1 * time.Minute)),
-		},
-	}
-	activities := []JobActivity{
-		{
-			JobID:      "job-1",
-			JobType:    "vacuum",
-			Source:     "worker_progress",
-			Message:    "processing",
-			Stage:      "running",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-			Details: map[string]interface{}{
-				"step": "scan",
-			},
-		},
-	}
-
-	if err := store.SaveTrackedJobs(tracked); err != nil {
-		t.Fatalf("SaveTrackedJobs: %v", err)
-	}
-	if err := store.SaveActivities(activities); err != nil {
-		t.Fatalf("SaveActivities: %v", err)
-	}
-
-	gotTracked, err := store.LoadTrackedJobs()
-	if err != nil {
-		t.Fatalf("LoadTrackedJobs: %v", err)
-	}
-	if len(gotTracked) != 1 || gotTracked[0].JobID != tracked[0].JobID {
-		t.Fatalf("unexpected tracked jobs: %+v", gotTracked)
-	}
-
-	gotActivities, err := store.LoadActivities()
-	if err != nil {
-		t.Fatalf("LoadActivities: %v", err)
-	}
-	if len(gotActivities) != 1 || gotActivities[0].Message != activities[0].Message {
-		t.Fatalf("unexpected activities: %+v", gotActivities)
-	}
-	if gotActivities[0].Details["step"] != "scan" {
-		t.Fatalf("unexpected activity details: %+v", gotActivities[0].Details)
-	}
-}
-
-func TestConfigStoreJobDetailRoundTrip(t *testing.T) {
-	t.Parallel()
-
-	store, err := NewConfigStore(t.TempDir())
-	if err != nil {
-		t.Fatalf("NewConfigStore: %v", err)
-	}
-
-	input := TrackedJob{
-		JobID:     "job-detail-1",
-		JobType:   "vacuum",
-		Summary:   "detail summary",
-		Detail:    "detail payload",
-		CreatedAt: timeToPtr(time.Now().UTC().Add(-2 * time.Minute)),
-		UpdatedAt: timeToPtr(time.Now().UTC()),
-		Parameters: map[string]interface{}{
-			"volume_id": map[string]interface{}{"int64_value": "3"},
-		},
-		Labels: map[string]string{
-			"source": "detector",
-		},
-		ResultOutputValues: map[string]interface{}{
-			"moved": map[string]interface{}{"bool_value": true},
-		},
-	}
-
-	if err := store.SaveJobDetail(input); err != nil {
-		t.Fatalf("SaveJobDetail: %v", err)
-	}
-
-	got, err := store.LoadJobDetail(input.JobID)
-	if err != nil {
-		t.Fatalf("LoadJobDetail: %v", err)
-	}
-	if got == nil {
-		t.Fatalf("LoadJobDetail returned nil")
-	}
-	if got.Detail != input.Detail {
-		t.Fatalf("unexpected detail: got=%q want=%q", got.Detail, input.Detail)
-	}
-	if got.Labels["source"] != "detector" {
-		t.Fatalf("unexpected labels: %+v", got.Labels)
-	}
-	if got.ResultOutputValues == nil {
-		t.Fatalf("expected result output values")
-	}
-}

weed/admin/plugin/job_execution_plan.go

@@ -1,231 +0,0 @@
-package plugin
-
-import (
-	"encoding/base64"
-	"sort"
-	"strconv"
-	"strings"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
-	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
-	"google.golang.org/protobuf/proto"
-)
-
-func enrichTrackedJobParameters(jobType string, parameters map[string]interface{}) map[string]interface{} {
-	if len(parameters) == 0 {
-		return parameters
-	}
-	if _, exists := parameters["execution_plan"]; exists {
-		return parameters
-	}
-
-	taskParams, ok := decodeTaskParamsFromPlainParameters(parameters)
-	if !ok || taskParams == nil {
-		return parameters
-	}
-
-	plan := buildExecutionPlan(strings.TrimSpace(jobType), taskParams)
-	if plan == nil {
-		return parameters
-	}
-
-	enriched := make(map[string]interface{}, len(parameters)+1)
-	for key, value := range parameters {
-		enriched[key] = value
-	}
-	enriched["execution_plan"] = plan
-	return enriched
-}
-
-func decodeTaskParamsFromPlainParameters(parameters map[string]interface{}) (*worker_pb.TaskParams, bool) {
-	rawField, ok := parameters["task_params_pb"]
-	if !ok || rawField == nil {
-		return nil, false
-	}
-
-	fieldMap, ok := rawField.(map[string]interface{})
-	if !ok {
-		return nil, false
-	}
-
-	bytesValue, _ := fieldMap["bytes_value"].(string)
-	bytesValue = strings.TrimSpace(bytesValue)
-	if bytesValue == "" {
-		return nil, false
-	}
-
-	payload, err := base64.StdEncoding.DecodeString(bytesValue)
-	if err != nil {
-		return nil, false
-	}
-
-	params := &worker_pb.TaskParams{}
-	if err := proto.Unmarshal(payload, params); err != nil {
-		return nil, false
-	}
-
-	return params, true
-}
-
-func buildExecutionPlan(jobType string, params *worker_pb.TaskParams) map[string]interface{} {
-	if params == nil {
-		return nil
-	}
-
-	normalizedJobType := strings.TrimSpace(jobType)
-	if normalizedJobType == "" && params.GetErasureCodingParams() != nil {
-		normalizedJobType = "erasure_coding"
-	}
-
-	switch normalizedJobType {
-	case "erasure_coding":
-		return buildErasureCodingExecutionPlan(params)
-	default:
-		return nil
-	}
-}
-
-func buildErasureCodingExecutionPlan(params *worker_pb.TaskParams) map[string]interface{} {
-	if params == nil {
-		return nil
-	}
-
-	ecParams := params.GetErasureCodingParams()
-	if ecParams == nil {
-		return nil
-	}
-
-	dataShards := int(ecParams.DataShards)
-	if dataShards <= 0 {
-		dataShards = int(erasure_coding.DataShardsCount)
-	}
-	parityShards := int(ecParams.ParityShards)
-	if parityShards <= 0 {
-		parityShards = int(erasure_coding.ParityShardsCount)
-	}
-	totalShards := dataShards + parityShards
-
-	sources := make([]map[string]interface{}, 0, len(params.Sources))
-	for _, source := range params.Sources {
-		if source == nil {
-			continue
-		}
-		sources = append(sources, buildExecutionEndpoint(
-			source.Node,
-			source.DataCenter,
-			source.Rack,
-			source.VolumeId,
-			source.ShardIds,
-			dataShards,
-		))
-	}
-
-	targets := make([]map[string]interface{}, 0, len(params.Targets))
-	shardAssignments := make([]map[string]interface{}, 0, totalShards)
-	for targetIndex, target := range params.Targets {
-		if target == nil {
-			continue
-		}
-
-		targets = append(targets, buildExecutionEndpoint(
-			target.Node,
-			target.DataCenter,
-			target.Rack,
-			target.VolumeId,
-			target.ShardIds,
-			dataShards,
-		))
-
-		for _, shardID := range normalizeShardIDs(target.ShardIds) {
-			kind, label := classifyShardID(shardID, dataShards)
-			shardAssignments = append(shardAssignments, map[string]interface{}{
-				"shard_id":           shardID,
-				"kind":               kind,
-				"label":              label,
-				"target_index":       targetIndex + 1,
-				"target_node":        strings.TrimSpace(target.Node),
-				"target_data_center": strings.TrimSpace(target.DataCenter),
-				"target_rack":        strings.TrimSpace(target.Rack),
-				"target_volume_id":   int(target.VolumeId),
-			})
-		}
-	}
-	sort.Slice(shardAssignments, func(i, j int) bool {
-		left, _ := shardAssignments[i]["shard_id"].(int)
-		right, _ := shardAssignments[j]["shard_id"].(int)
-		return left < right
-	})
-
-	plan := map[string]interface{}{
-		"job_type":      "erasure_coding",
-		"task_id":       strings.TrimSpace(params.TaskId),
-		"volume_id":     int(params.VolumeId),
-		"collection":    strings.TrimSpace(params.Collection),
-		"data_shards":   dataShards,
-		"parity_shards": parityShards,
-		"total_shards":  totalShards,
-		"sources":       sources,
-		"targets":       targets,
-		"source_count":  len(sources),
-		"target_count":  len(targets),
-	}
-
-	if len(shardAssignments) > 0 {
-		plan["shard_assignments"] = shardAssignments
-	}
-
-	return plan
-}
-
-func buildExecutionEndpoint(
-	node string,
-	dataCenter string,
-	rack string,
-	volumeID uint32,
-	shardIDs []uint32,
-	dataShardCount int,
-) map[string]interface{} {
-	allShards := normalizeShardIDs(shardIDs)
-	dataShards := make([]int, 0, len(allShards))
-	parityShards := make([]int, 0, len(allShards))
-	for _, shardID := range allShards {
-		if shardID < dataShardCount {
-			dataShards = append(dataShards, shardID)
-		} else {
-			parityShards = append(parityShards, shardID)
-		}
-	}
-
-	return map[string]interface{}{
-		"node":             strings.TrimSpace(node),
-		"data_center":      strings.TrimSpace(dataCenter),
-		"rack":             strings.TrimSpace(rack),
-		"volume_id":        int(volumeID),
-		"shard_ids":        allShards,
-		"data_shard_ids":   dataShards,
-		"parity_shard_ids": parityShards,
-	}
-}
-
-func normalizeShardIDs(shardIDs []uint32) []int {
-	if len(shardIDs) == 0 {
-		return nil
-	}
-
-	out := make([]int, 0, len(shardIDs))
-	for _, shardID := range shardIDs {
-		out = append(out, int(shardID))
-	}
-	sort.Ints(out)
-	return out
-}
-
-func classifyShardID(shardID int, dataShardCount int) (kind string, label string) {
-	if dataShardCount <= 0 {
-		dataShardCount = int(erasure_coding.DataShardsCount)
-	}
-	if shardID < dataShardCount {
-		return "data", "D" + strconv.Itoa(shardID)
-	}
-	return "parity", "P" + strconv.Itoa(shardID)
-}

weed/admin/plugin/plugin_cancel_test.go

@@ -1,112 +0,0 @@
-package plugin
-
-import (
-	"context"
-	"errors"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-)
-
-func TestRunDetectionSendsCancelOnContextDone(t *testing.T) {
-	t.Parallel()
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New plugin error: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	const workerID = "worker-detect"
-	const jobType = "vacuum"
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: workerID,
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
-		},
-	})
-	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4)}
-	pluginSvc.putSession(session)
-
-	ctx, cancel := context.WithCancel(context.Background())
-	errCh := make(chan error, 1)
-	go func() {
-		_, runErr := pluginSvc.RunDetection(ctx, jobType, &plugin_pb.ClusterContext{}, 10)
-		errCh <- runErr
-	}()
-
-	first := <-session.outgoing
-	if first.GetRunDetectionRequest() == nil {
-		t.Fatalf("expected first message to be run_detection_request")
-	}
-
-	cancel()
-
-	second := <-session.outgoing
-	cancelReq := second.GetCancelRequest()
-	if cancelReq == nil {
-		t.Fatalf("expected second message to be cancel_request")
-	}
-	if cancelReq.TargetId != first.RequestId {
-		t.Fatalf("unexpected cancel target id: got=%s want=%s", cancelReq.TargetId, first.RequestId)
-	}
-	if cancelReq.TargetKind != plugin_pb.WorkKind_WORK_KIND_DETECTION {
-		t.Fatalf("unexpected cancel target kind: %v", cancelReq.TargetKind)
-	}
-
-	runErr := <-errCh
-	if !errors.Is(runErr, context.Canceled) {
-		t.Fatalf("expected context canceled error, got %v", runErr)
-	}
-}
-
-func TestExecuteJobSendsCancelOnContextDone(t *testing.T) {
-	t.Parallel()
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New plugin error: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	const workerID = "worker-exec"
-	const jobType = "vacuum"
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: workerID,
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: jobType, CanExecute: true, MaxExecutionConcurrency: 1},
-		},
-	})
-	session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4)}
-	pluginSvc.putSession(session)
-
-	job := &plugin_pb.JobSpec{JobId: "job-1", JobType: jobType}
-	ctx, cancel := context.WithCancel(context.Background())
-	errCh := make(chan error, 1)
-	go func() {
-		_, runErr := pluginSvc.ExecuteJob(ctx, job, &plugin_pb.ClusterContext{}, 1)
-		errCh <- runErr
-	}()
-
-	first := <-session.outgoing
-	if first.GetExecuteJobRequest() == nil {
-		t.Fatalf("expected first message to be execute_job_request")
-	}
-
-	cancel()
-
-	second := <-session.outgoing
-	cancelReq := second.GetCancelRequest()
-	if cancelReq == nil {
-		t.Fatalf("expected second message to be cancel_request")
-	}
-	if cancelReq.TargetId != first.RequestId {
-		t.Fatalf("unexpected cancel target id: got=%s want=%s", cancelReq.TargetId, first.RequestId)
-	}
-	if cancelReq.TargetKind != plugin_pb.WorkKind_WORK_KIND_EXECUTION {
-		t.Fatalf("unexpected cancel target kind: %v", cancelReq.TargetKind)
-	}
-
-	runErr := <-errCh
-	if !errors.Is(runErr, context.Canceled) {
-		t.Fatalf("expected context canceled error, got %v", runErr)
-	}
-}

weed/admin/plugin/plugin_config_bootstrap_test.go

@@ -1,125 +0,0 @@
-package plugin
-
-import (
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-)
-
-func TestEnsureJobTypeConfigFromDescriptorBootstrapsDefaults(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	descriptor := &plugin_pb.JobTypeDescriptor{
-		JobType:           "vacuum",
-		DescriptorVersion: 3,
-		AdminConfigForm: &plugin_pb.ConfigForm{
-			DefaultValues: map[string]*plugin_pb.ConfigValue{
-				"scan_scope": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "all"}},
-			},
-		},
-		WorkerConfigForm: &plugin_pb.ConfigForm{
-			DefaultValues: map[string]*plugin_pb.ConfigValue{
-				"threshold": {Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.3}},
-			},
-		},
-		AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
-			Enabled:                       true,
-			DetectionIntervalSeconds:      60,
-			DetectionTimeoutSeconds:       20,
-			MaxJobsPerDetection:           30,
-			GlobalExecutionConcurrency:    4,
-			PerWorkerExecutionConcurrency: 2,
-			RetryLimit:                    3,
-			RetryBackoffSeconds:           5,
-		},
-	}
-
-	if err := pluginSvc.ensureJobTypeConfigFromDescriptor("vacuum", descriptor); err != nil {
-		t.Fatalf("ensureJobTypeConfigFromDescriptor: %v", err)
-	}
-
-	cfg, err := pluginSvc.LoadJobTypeConfig("vacuum")
-	if err != nil {
-		t.Fatalf("LoadJobTypeConfig: %v", err)
-	}
-	if cfg == nil {
-		t.Fatalf("expected non-nil config")
-	}
-	if cfg.DescriptorVersion != 3 {
-		t.Fatalf("unexpected descriptor version: got=%d", cfg.DescriptorVersion)
-	}
-	if cfg.AdminRuntime == nil || !cfg.AdminRuntime.Enabled {
-		t.Fatalf("expected enabled admin settings")
-	}
-	if cfg.AdminRuntime.GlobalExecutionConcurrency != 4 {
-		t.Fatalf("unexpected global execution concurrency: %d", cfg.AdminRuntime.GlobalExecutionConcurrency)
-	}
-	if _, ok := cfg.AdminConfigValues["scan_scope"]; !ok {
-		t.Fatalf("missing admin default value")
-	}
-	if _, ok := cfg.WorkerConfigValues["threshold"]; !ok {
-		t.Fatalf("missing worker default value")
-	}
-}
-
-func TestEnsureJobTypeConfigFromDescriptorDoesNotOverwriteExisting(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	if err := pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
-		JobType: "balance",
-		AdminRuntime: &plugin_pb.AdminRuntimeConfig{
-			Enabled:                    true,
-			GlobalExecutionConcurrency: 9,
-		},
-		AdminConfigValues: map[string]*plugin_pb.ConfigValue{
-			"custom": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "keep"}},
-		},
-	}); err != nil {
-		t.Fatalf("SaveJobTypeConfig: %v", err)
-	}
-
-	descriptor := &plugin_pb.JobTypeDescriptor{
-		JobType:           "balance",
-		DescriptorVersion: 7,
-		AdminConfigForm: &plugin_pb.ConfigForm{
-			DefaultValues: map[string]*plugin_pb.ConfigValue{
-				"custom": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "overwrite"}},
-			},
-		},
-		AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
-			Enabled:                    true,
-			GlobalExecutionConcurrency: 1,
-		},
-	}
-
-	if err := pluginSvc.ensureJobTypeConfigFromDescriptor("balance", descriptor); err != nil {
-		t.Fatalf("ensureJobTypeConfigFromDescriptor: %v", err)
-	}
-
-	cfg, err := pluginSvc.LoadJobTypeConfig("balance")
-	if err != nil {
-		t.Fatalf("LoadJobTypeConfig: %v", err)
-	}
-	if cfg == nil {
-		t.Fatalf("expected config")
-	}
-	if cfg.AdminRuntime == nil || cfg.AdminRuntime.GlobalExecutionConcurrency != 9 {
-		t.Fatalf("existing admin settings should be preserved, got=%v", cfg.AdminRuntime)
-	}
-	custom := cfg.AdminConfigValues["custom"]
-	if custom == nil || custom.GetStringValue() != "keep" {
-		t.Fatalf("existing admin config should be preserved")
-	}
-}

weed/admin/plugin/plugin_detection_test.go

@@ -1,197 +0,0 @@
-package plugin
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-)
-
-func TestRunDetectionIncludesLatestSuccessfulRun(t *testing.T) {
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New plugin error: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	jobType := "vacuum"
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
-		},
-	})
-	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
-	pluginSvc.putSession(session)
-
-	oldSuccess := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
-	latestSuccess := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC)
-	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{Outcome: RunOutcomeSuccess, CompletedAt: timeToPtr(oldSuccess)}); err != nil {
-		t.Fatalf("AppendRunRecord old success: %v", err)
-	}
-	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{Outcome: RunOutcomeError, CompletedAt: timeToPtr(latestSuccess.Add(2 * time.Hour))}); err != nil {
-		t.Fatalf("AppendRunRecord error run: %v", err)
-	}
-	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{Outcome: RunOutcomeSuccess, CompletedAt: timeToPtr(latestSuccess)}); err != nil {
-		t.Fatalf("AppendRunRecord latest success: %v", err)
-	}
-
-	resultCh := make(chan error, 1)
-	go func() {
-		_, runErr := pluginSvc.RunDetection(context.Background(), jobType, &plugin_pb.ClusterContext{}, 10)
-		resultCh <- runErr
-	}()
-
-	message := <-session.outgoing
-	detectRequest := message.GetRunDetectionRequest()
-	if detectRequest == nil {
-		t.Fatalf("expected run detection request message")
-	}
-	if detectRequest.LastSuccessfulRun == nil {
-		t.Fatalf("expected last_successful_run to be set")
-	}
-	if got := detectRequest.LastSuccessfulRun.AsTime().UTC(); !got.Equal(latestSuccess) {
-		t.Fatalf("unexpected last_successful_run, got=%s want=%s", got, latestSuccess)
-	}
-
-	pluginSvc.handleDetectionComplete("worker-a", &plugin_pb.DetectionComplete{
-		RequestId: message.RequestId,
-		JobType:   jobType,
-		Success:   true,
-	})
-
-	if runErr := <-resultCh; runErr != nil {
-		t.Fatalf("RunDetection error: %v", runErr)
-	}
-}
-
-func TestRunDetectionOmitsLastSuccessfulRunWhenNoSuccessHistory(t *testing.T) {
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New plugin error: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	jobType := "vacuum"
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
-		},
-	})
-	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
-	pluginSvc.putSession(session)
-
-	if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
-		Outcome:     RunOutcomeError,
-		CompletedAt: timeToPtr(time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC)),
-	}); err != nil {
-		t.Fatalf("AppendRunRecord error run: %v", err)
-	}
-
-	resultCh := make(chan error, 1)
-	go func() {
-		_, runErr := pluginSvc.RunDetection(context.Background(), jobType, &plugin_pb.ClusterContext{}, 10)
-		resultCh <- runErr
-	}()
-
-	message := <-session.outgoing
-	detectRequest := message.GetRunDetectionRequest()
-	if detectRequest == nil {
-		t.Fatalf("expected run detection request message")
-	}
-	if detectRequest.LastSuccessfulRun != nil {
-		t.Fatalf("expected last_successful_run to be nil when no success history")
-	}
-
-	pluginSvc.handleDetectionComplete("worker-a", &plugin_pb.DetectionComplete{
-		RequestId: message.RequestId,
-		JobType:   jobType,
-		Success:   true,
-	})
-
-	if runErr := <-resultCh; runErr != nil {
-		t.Fatalf("RunDetection error: %v", runErr)
-	}
-}
-
-func TestRunDetectionWithReportCapturesDetectionActivities(t *testing.T) {
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New plugin error: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	jobType := "vacuum"
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
-		},
-	})
-	session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
-	pluginSvc.putSession(session)
-
-	reportCh := make(chan *DetectionReport, 1)
-	errCh := make(chan error, 1)
-	go func() {
-		report, runErr := pluginSvc.RunDetectionWithReport(context.Background(), jobType, &plugin_pb.ClusterContext{}, 10)
-		reportCh <- report
-		errCh <- runErr
-	}()
-
-	message := <-session.outgoing
-	requestID := message.GetRequestId()
-	if requestID == "" {
-		t.Fatalf("expected request id in detection request")
-	}
-
-	pluginSvc.handleDetectionProposals("worker-a", &plugin_pb.DetectionProposals{
-		RequestId: requestID,
-		JobType:   jobType,
-		Proposals: []*plugin_pb.JobProposal{
-			{
-				ProposalId: "proposal-1",
-				JobType:    jobType,
-				Summary:    "vacuum proposal",
-				Detail:     "based on garbage ratio",
-			},
-		},
-	})
-	pluginSvc.handleDetectionComplete("worker-a", &plugin_pb.DetectionComplete{
-		RequestId:      requestID,
-		JobType:        jobType,
-		Success:        true,
-		TotalProposals: 1,
-	})
-
-	report := <-reportCh
-	if report == nil {
-		t.Fatalf("expected detection report")
-	}
-	if report.RequestID == "" {
-		t.Fatalf("expected detection report request id")
-	}
-	if report.WorkerID != "worker-a" {
-		t.Fatalf("expected worker-a, got %q", report.WorkerID)
-	}
-	if len(report.Proposals) != 1 {
-		t.Fatalf("expected one proposal in report, got %d", len(report.Proposals))
-	}
-	if runErr := <-errCh; runErr != nil {
-		t.Fatalf("RunDetectionWithReport error: %v", runErr)
-	}
-
-	activities := pluginSvc.ListActivities(jobType, 0)
-	stages := map[string]bool{}
-	for _, activity := range activities {
-		if activity.RequestID != report.RequestID {
-			continue
-		}
-		stages[activity.Stage] = true
-	}
-	if !stages["requested"] || !stages["proposal"] || !stages["completed"] {
-		t.Fatalf("expected requested/proposal/completed activities, got stages=%v", stages)
-	}
-}

weed/admin/plugin/plugin_monitor.go

@@ -1,896 +0,0 @@
-package plugin
-
-import (
-	"encoding/json"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-	"google.golang.org/protobuf/encoding/protojson"
-)
-
-const (
-	maxTrackedJobsTotal = 1000
-	maxActivityRecords  = 4000
-	maxRelatedJobs      = 100
-)
-
-var (
-	StateSucceeded = strings.ToLower(plugin_pb.JobState_JOB_STATE_SUCCEEDED.String())
-	StateFailed    = strings.ToLower(plugin_pb.JobState_JOB_STATE_FAILED.String())
-	StateCanceled  = strings.ToLower(plugin_pb.JobState_JOB_STATE_CANCELED.String())
-)
-
-// activityLess reports whether activity a occurred after activity b (newest-first order).
-// A nil OccurredAt is treated as the zero time.
-func activityLess(a, b JobActivity) bool {
-	ta := time.Time{}
-	if a.OccurredAt != nil {
-		ta = *a.OccurredAt
-	}
-	tb := time.Time{}
-	if b.OccurredAt != nil {
-		tb = *b.OccurredAt
-	}
-	return ta.After(tb)
-}
-
-func (r *Plugin) loadPersistedMonitorState() error {
-	trackedJobs, err := r.store.LoadTrackedJobs()
-	if err != nil {
-		return err
-	}
-	activities, err := r.store.LoadActivities()
-	if err != nil {
-		return err
-	}
-
-	if len(trackedJobs) > 0 {
-		r.jobsMu.Lock()
-		for i := range trackedJobs {
-			job := trackedJobs[i]
-			if strings.TrimSpace(job.JobID) == "" {
-				continue
-			}
-			// Backward compatibility: migrate older inline detail payloads
-			// out of tracked_jobs.json into dedicated per-job detail files.
-			if hasTrackedJobRichDetails(job) {
-				if err := r.store.SaveJobDetail(job); err != nil {
-					glog.Warningf("Plugin failed to migrate detail snapshot for job %s: %v", job.JobID, err)
-				}
-			}
-			stripTrackedJobDetailFields(&job)
-			jobCopy := job
-			r.jobs[job.JobID] = &jobCopy
-		}
-		r.pruneTrackedJobsLocked()
-		r.jobsMu.Unlock()
-	}
-
-	if len(activities) > maxActivityRecords {
-		activities = activities[len(activities)-maxActivityRecords:]
-	}
-	if len(activities) > 0 {
-		r.activitiesMu.Lock()
-		r.activities = append([]JobActivity(nil), activities...)
-		r.activitiesMu.Unlock()
-	}
-
-	return nil
-}
-
-func (r *Plugin) ListTrackedJobs(jobType string, state string, limit int) []TrackedJob {
-	r.jobsMu.RLock()
-	defer r.jobsMu.RUnlock()
-
-	normalizedJobType := strings.TrimSpace(jobType)
-	normalizedState := strings.TrimSpace(strings.ToLower(state))
-
-	items := make([]TrackedJob, 0, len(r.jobs))
-	for _, job := range r.jobs {
-		if job == nil {
-			continue
-		}
-		if normalizedJobType != "" && job.JobType != normalizedJobType {
-			continue
-		}
-		if normalizedState != "" && strings.ToLower(job.State) != normalizedState {
-			continue
-		}
-		items = append(items, cloneTrackedJob(*job))
-	}
-
-	sort.Slice(items, func(i, j int) bool {
-		ti := time.Time{}
-		if items[i].UpdatedAt != nil {
-			ti = *items[i].UpdatedAt
-		}
-		tj := time.Time{}
-		if items[j].UpdatedAt != nil {
-			tj = *items[j].UpdatedAt
-		}
-		if !ti.Equal(tj) {
-			return ti.After(tj)
-		}
-		return items[i].JobID < items[j].JobID
-	})
-
-	if limit > 0 && len(items) > limit {
-		items = items[:limit]
-	}
-	return items
-}
-
-func (r *Plugin) GetTrackedJob(jobID string) (*TrackedJob, bool) {
-	r.jobsMu.RLock()
-	defer r.jobsMu.RUnlock()
-
-	job, ok := r.jobs[jobID]
-	if !ok || job == nil {
-		return nil, false
-	}
-	clone := cloneTrackedJob(*job)
-	return &clone, true
-}
-
-func (r *Plugin) ListActivities(jobType string, limit int) []JobActivity {
-	r.activitiesMu.RLock()
-	defer r.activitiesMu.RUnlock()
-
-	normalized := strings.TrimSpace(jobType)
-	activities := make([]JobActivity, 0, len(r.activities))
-	for _, activity := range r.activities {
-		if normalized != "" && activity.JobType != normalized {
-			continue
-		}
-		activities = append(activities, activity)
-	}
-
-	sort.Slice(activities, func(i, j int) bool {
-		return activityLess(activities[i], activities[j])
-	})
-	if limit > 0 && len(activities) > limit {
-		activities = activities[:limit]
-	}
-	return activities
-}
-
-func (r *Plugin) ListJobActivities(jobID string, limit int) []JobActivity {
-	normalizedJobID := strings.TrimSpace(jobID)
-	if normalizedJobID == "" {
-		return nil
-	}
-
-	r.activitiesMu.RLock()
-	activities := make([]JobActivity, 0, len(r.activities))
-	for _, activity := range r.activities {
-		if strings.TrimSpace(activity.JobID) != normalizedJobID {
-			continue
-		}
-		activities = append(activities, activity)
-	}
-	r.activitiesMu.RUnlock()
-
-	sort.Slice(activities, func(i, j int) bool {
-		return !activityLess(activities[i], activities[j]) // oldest-first for job timeline
-	})
-	if limit > 0 && len(activities) > limit {
-		activities = activities[len(activities)-limit:]
-	}
-	return activities
-}
-
-func (r *Plugin) BuildJobDetail(jobID string, activityLimit int, relatedLimit int) (*JobDetail, bool, error) {
-	normalizedJobID := strings.TrimSpace(jobID)
-	if normalizedJobID == "" {
-		return nil, false, nil
-	}
-
-	// Clamp relatedLimit to a safe range to avoid excessive memory allocation from untrusted input.
-	if relatedLimit <= 0 {
-		relatedLimit = 0
-	} else if relatedLimit > maxRelatedJobs {
-		relatedLimit = maxRelatedJobs
-	}
-
-	r.jobsMu.RLock()
-	trackedSnapshot, ok := r.jobs[normalizedJobID]
-	if ok && trackedSnapshot != nil {
-		candidate := cloneTrackedJob(*trackedSnapshot)
-		stripTrackedJobDetailFields(&candidate)
-		trackedSnapshot = &candidate
-	} else {
-		trackedSnapshot = nil
-	}
-	r.jobsMu.RUnlock()
-
-	detailJob, err := r.store.LoadJobDetail(normalizedJobID)
-	if err != nil {
-		return nil, false, err
-	}
-
-	if trackedSnapshot == nil && detailJob == nil {
-		return nil, false, nil
-	}
-	if detailJob == nil && trackedSnapshot != nil {
-		clone := cloneTrackedJob(*trackedSnapshot)
-		detailJob = &clone
-	}
-	if detailJob == nil {
-		return nil, false, nil
-	}
-	if trackedSnapshot != nil {
-		mergeTrackedStatusIntoDetail(detailJob, trackedSnapshot)
-	}
-	detailJob.Parameters = enrichTrackedJobParameters(detailJob.JobType, detailJob.Parameters)
-
-	r.activitiesMu.RLock()
-	activities := append([]JobActivity(nil), r.activities...)
-	r.activitiesMu.RUnlock()
-
-	detail := &JobDetail{
-		Job:         detailJob,
-		Activities:  filterJobActivitiesFromSlice(activities, normalizedJobID, activityLimit),
-		LastUpdated: timeToPtr(time.Now().UTC()),
-	}
-
-	if history, err := r.store.LoadRunHistory(detailJob.JobType); err != nil {
-		return nil, true, err
-	} else if history != nil {
-		for i := range history.SuccessfulRuns {
-			record := history.SuccessfulRuns[i]
-			if strings.TrimSpace(record.JobID) == normalizedJobID {
-				recordCopy := record
-				detail.RunRecord = &recordCopy
-				break
-			}
-		}
-		if detail.RunRecord == nil {
-			for i := range history.ErrorRuns {
-				record := history.ErrorRuns[i]
-				if strings.TrimSpace(record.JobID) == normalizedJobID {
-					recordCopy := record
-					detail.RunRecord = &recordCopy
-					break
-				}
-			}
-		}
-	}
-
-	if relatedLimit > 0 {
-		related := make([]TrackedJob, 0, relatedLimit)
-		r.jobsMu.RLock()
-		for _, candidate := range r.jobs {
-			if strings.TrimSpace(candidate.JobType) != strings.TrimSpace(detailJob.JobType) {
-				continue
-			}
-			if strings.TrimSpace(candidate.JobID) == normalizedJobID {
-				continue
-			}
-			cloned := cloneTrackedJob(*candidate)
-			stripTrackedJobDetailFields(&cloned)
-			related = append(related, cloned)
-			if len(related) >= relatedLimit {
-				break
-			}
-		}
-		r.jobsMu.RUnlock()
-		detail.RelatedJobs = related
-	}
-
-	return detail, true, nil
-}
-
-func filterJobActivitiesFromSlice(all []JobActivity, jobID string, limit int) []JobActivity {
-	if strings.TrimSpace(jobID) == "" || len(all) == 0 {
-		return nil
-	}
-
-	activities := make([]JobActivity, 0, len(all))
-	for _, activity := range all {
-		if strings.TrimSpace(activity.JobID) != jobID {
-			continue
-		}
-		activities = append(activities, activity)
-	}
-
-	sort.Slice(activities, func(i, j int) bool {
-		return !activityLess(activities[i], activities[j]) // oldest-first for job timeline
-	})
-	if limit > 0 && len(activities) > limit {
-		activities = activities[len(activities)-limit:]
-	}
-	return activities
-}
-
-func stripTrackedJobDetailFields(job *TrackedJob) {
-	if job == nil {
-		return
-	}
-	job.Detail = ""
-	job.Parameters = nil
-	job.Labels = nil
-	job.ResultOutputValues = nil
-}
-
-func hasTrackedJobRichDetails(job TrackedJob) bool {
-	return strings.TrimSpace(job.Detail) != "" ||
-		len(job.Parameters) > 0 ||
-		len(job.Labels) > 0 ||
-		len(job.ResultOutputValues) > 0
-}
-
-func mergeTrackedStatusIntoDetail(detail *TrackedJob, tracked *TrackedJob) {
-	if detail == nil || tracked == nil {
-		return
-	}
-
-	if detail.JobType == "" {
-		detail.JobType = tracked.JobType
-	}
-	if detail.RequestID == "" {
-		detail.RequestID = tracked.RequestID
-	}
-	if detail.WorkerID == "" {
-		detail.WorkerID = tracked.WorkerID
-	}
-	if detail.DedupeKey == "" {
-		detail.DedupeKey = tracked.DedupeKey
-	}
-	if detail.Summary == "" {
-		detail.Summary = tracked.Summary
-	}
-	if detail.State == "" {
-		detail.State = tracked.State
-	}
-	if detail.Progress == 0 {
-		detail.Progress = tracked.Progress
-	}
-	if detail.Stage == "" {
-		detail.Stage = tracked.Stage
-	}
-	if detail.Message == "" {
-		detail.Message = tracked.Message
-	}
-	if detail.Attempt == 0 {
-		detail.Attempt = tracked.Attempt
-	}
-	if detail.CreatedAt == nil || detail.CreatedAt.IsZero() {
-		detail.CreatedAt = tracked.CreatedAt
-	}
-	if detail.UpdatedAt == nil || detail.UpdatedAt.IsZero() {
-		detail.UpdatedAt = tracked.UpdatedAt
-	}
-	if detail.CompletedAt == nil || detail.CompletedAt.IsZero() {
-		detail.CompletedAt = tracked.CompletedAt
-	}
-	if detail.ErrorMessage == "" {
-		detail.ErrorMessage = tracked.ErrorMessage
-	}
-	if detail.ResultSummary == "" {
-		detail.ResultSummary = tracked.ResultSummary
-	}
-}
-
-func (r *Plugin) handleJobProgressUpdate(workerID string, update *plugin_pb.JobProgressUpdate) {
-	if update == nil {
-		return
-	}
-
-	now := time.Now().UTC()
-	resolvedWorkerID := strings.TrimSpace(workerID)
-
-	if strings.TrimSpace(update.JobId) != "" {
-		r.jobsMu.Lock()
-		job := r.jobs[update.JobId]
-		if job == nil {
-			job = &TrackedJob{
-				JobID:     update.JobId,
-				JobType:   update.JobType,
-				RequestID: update.RequestId,
-				WorkerID:  resolvedWorkerID,
-				CreatedAt: timeToPtr(now),
-			}
-			r.jobs[update.JobId] = job
-		}
-
-		if update.JobType != "" {
-			job.JobType = update.JobType
-		}
-		if update.RequestId != "" {
-			job.RequestID = update.RequestId
-		}
-		if job.WorkerID != "" {
-			resolvedWorkerID = job.WorkerID
-		} else if resolvedWorkerID != "" {
-			job.WorkerID = resolvedWorkerID
-		}
-		job.State = strings.ToLower(update.State.String())
-		job.Progress = update.ProgressPercent
-		job.Stage = update.Stage
-		job.Message = update.Message
-		job.UpdatedAt = timeToPtr(now)
-		r.pruneTrackedJobsLocked()
-		r.dirtyJobs = true
-		r.jobsMu.Unlock()
-	}
-
-	r.trackWorkerActivities(update.JobType, update.JobId, update.RequestId, resolvedWorkerID, update.Activities)
-	if update.Message != "" || update.Stage != "" {
-		source := "worker_progress"
-		if strings.TrimSpace(update.JobId) == "" {
-			source = "worker_detection"
-		}
-		r.appendActivity(JobActivity{
-			JobID:      update.JobId,
-			JobType:    update.JobType,
-			RequestID:  update.RequestId,
-			WorkerID:   resolvedWorkerID,
-			Source:     source,
-			Message:    update.Message,
-			Stage:      update.Stage,
-			OccurredAt: timeToPtr(now),
-		})
-	}
-}
-
-func (r *Plugin) trackExecutionStart(requestID, workerID string, job *plugin_pb.JobSpec, attempt int32) {
-	if job == nil || strings.TrimSpace(job.JobId) == "" {
-		return
-	}
-
-	now := time.Now().UTC()
-
-	r.jobsMu.Lock()
-	tracked := r.jobs[job.JobId]
-	if tracked == nil {
-		tracked = &TrackedJob{
-			JobID:     job.JobId,
-			CreatedAt: timeToPtr(now),
-		}
-		r.jobs[job.JobId] = tracked
-	}
-
-	tracked.JobType = job.JobType
-	tracked.RequestID = requestID
-	tracked.WorkerID = workerID
-	tracked.DedupeKey = job.DedupeKey
-	tracked.Summary = job.Summary
-	tracked.State = strings.ToLower(plugin_pb.JobState_JOB_STATE_ASSIGNED.String())
-	tracked.Progress = 0
-	tracked.Stage = "assigned"
-	tracked.Message = "job assigned to worker"
-	tracked.Attempt = attempt
-	if tracked.CreatedAt == nil || tracked.CreatedAt.IsZero() {
-		tracked.CreatedAt = timeToPtr(now)
-	}
-	tracked.UpdatedAt = timeToPtr(now)
-	trackedSnapshot := cloneTrackedJob(*tracked)
-	r.pruneTrackedJobsLocked()
-	r.dirtyJobs = true
-	r.jobsMu.Unlock()
-	r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) {
-		detail.JobID = job.JobId
-		detail.JobType = job.JobType
-		detail.RequestID = requestID
-		detail.WorkerID = workerID
-		detail.DedupeKey = job.DedupeKey
-		detail.Summary = job.Summary
-		detail.Detail = job.Detail
-		detail.Parameters = enrichTrackedJobParameters(job.JobType, configValueMapToPlain(job.Parameters))
-		if len(job.Labels) > 0 {
-			labels := make(map[string]string, len(job.Labels))
-			for key, value := range job.Labels {
-				labels[key] = value
-			}
-			detail.Labels = labels
-		} else {
-			detail.Labels = nil
-		}
-		detail.State = trackedSnapshot.State
-		detail.Progress = trackedSnapshot.Progress
-		detail.Stage = trackedSnapshot.Stage
-		detail.Message = trackedSnapshot.Message
-		detail.Attempt = attempt
-		if detail.CreatedAt == nil || detail.CreatedAt.IsZero() {
-			detail.CreatedAt = trackedSnapshot.CreatedAt
-		}
-		detail.UpdatedAt = trackedSnapshot.UpdatedAt
-	})
-
-	r.appendActivity(JobActivity{
-		JobID:      job.JobId,
-		JobType:    job.JobType,
-		RequestID:  requestID,
-		WorkerID:   workerID,
-		Source:     "admin_dispatch",
-		Message:    "job assigned",
-		Stage:      "assigned",
-		OccurredAt: timeToPtr(now),
-	})
-}
-
-func (r *Plugin) trackExecutionQueued(job *plugin_pb.JobSpec) {
-	if job == nil || strings.TrimSpace(job.JobId) == "" {
-		return
-	}
-
-	now := time.Now().UTC()
-
-	r.jobsMu.Lock()
-	tracked := r.jobs[job.JobId]
-	if tracked == nil {
-		tracked = &TrackedJob{
-			JobID:     job.JobId,
-			CreatedAt: timeToPtr(now),
-		}
-		r.jobs[job.JobId] = tracked
-	}
-
-	tracked.JobType = job.JobType
-	tracked.DedupeKey = job.DedupeKey
-	tracked.Summary = job.Summary
-	tracked.State = strings.ToLower(plugin_pb.JobState_JOB_STATE_PENDING.String())
-	tracked.Progress = 0
-	tracked.Stage = "queued"
-	tracked.Message = "waiting for available executor"
-	if tracked.CreatedAt == nil || tracked.CreatedAt.IsZero() {
-		tracked.CreatedAt = timeToPtr(now)
-	}
-	tracked.UpdatedAt = timeToPtr(now)
-	trackedSnapshot := cloneTrackedJob(*tracked)
-	r.pruneTrackedJobsLocked()
-	r.dirtyJobs = true
-	r.jobsMu.Unlock()
-	r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) {
-		detail.JobID = job.JobId
-		detail.JobType = job.JobType
-		detail.DedupeKey = job.DedupeKey
-		detail.Summary = job.Summary
-		detail.Detail = job.Detail
-		detail.Parameters = enrichTrackedJobParameters(job.JobType, configValueMapToPlain(job.Parameters))
-		if len(job.Labels) > 0 {
-			labels := make(map[string]string, len(job.Labels))
-			for key, value := range job.Labels {
-				labels[key] = value
-			}
-			detail.Labels = labels
-		} else {
-			detail.Labels = nil
-		}
-		detail.State = trackedSnapshot.State
-		detail.Progress = trackedSnapshot.Progress
-		detail.Stage = trackedSnapshot.Stage
-		detail.Message = trackedSnapshot.Message
-		if detail.CreatedAt == nil || detail.CreatedAt.IsZero() {
-			detail.CreatedAt = trackedSnapshot.CreatedAt
-		}
-		detail.UpdatedAt = trackedSnapshot.UpdatedAt
-	})
-
-	r.appendActivity(JobActivity{
-		JobID:      job.JobId,
-		JobType:    job.JobType,
-		Source:     "admin_scheduler",
-		Message:    "job queued for execution",
-		Stage:      "queued",
-		OccurredAt: timeToPtr(now),
-	})
-}
-
-func (r *Plugin) trackExecutionCompletion(completed *plugin_pb.JobCompleted) *TrackedJob {
-	if completed == nil || strings.TrimSpace(completed.JobId) == "" {
-		return nil
-	}
-
-	now := time.Now().UTC()
-	if completed.CompletedAt != nil {
-		now = completed.CompletedAt.AsTime().UTC()
-	}
-
-	r.jobsMu.Lock()
-	tracked := r.jobs[completed.JobId]
-	if tracked == nil {
-		tracked = &TrackedJob{
-			JobID:     completed.JobId,
-			CreatedAt: timeToPtr(now),
-		}
-		r.jobs[completed.JobId] = tracked
-	}
-
-	if completed.JobType != "" {
-		tracked.JobType = completed.JobType
-	}
-	if completed.RequestId != "" {
-		tracked.RequestID = completed.RequestId
-	}
-	if completed.Success {
-		tracked.State = strings.ToLower(plugin_pb.JobState_JOB_STATE_SUCCEEDED.String())
-		tracked.Progress = 100
-		tracked.Stage = "completed"
-		if completed.Result != nil {
-			tracked.ResultSummary = completed.Result.Summary
-		}
-		tracked.Message = tracked.ResultSummary
-		if tracked.Message == "" {
-			tracked.Message = "completed"
-		}
-		tracked.ErrorMessage = ""
-	} else {
-		tracked.State = strings.ToLower(plugin_pb.JobState_JOB_STATE_FAILED.String())
-		tracked.Stage = "failed"
-		tracked.ErrorMessage = completed.ErrorMessage
-		tracked.Message = completed.ErrorMessage
-	}
-
-	tracked.UpdatedAt = timeToPtr(now)
-	tracked.CompletedAt = timeToPtr(now)
-	r.pruneTrackedJobsLocked()
-	clone := cloneTrackedJob(*tracked)
-	r.dirtyJobs = true
-	r.jobsMu.Unlock()
-	r.persistJobDetailSnapshot(completed.JobId, func(detail *TrackedJob) {
-		detail.JobID = completed.JobId
-		if completed.JobType != "" {
-			detail.JobType = completed.JobType
-		}
-		if completed.RequestId != "" {
-			detail.RequestID = completed.RequestId
-		}
-		detail.State = clone.State
-		detail.Progress = clone.Progress
-		detail.Stage = clone.Stage
-		detail.Message = clone.Message
-		detail.ErrorMessage = clone.ErrorMessage
-		detail.ResultSummary = clone.ResultSummary
-		if completed.Success && completed.Result != nil {
-			detail.ResultOutputValues = configValueMapToPlain(completed.Result.OutputValues)
-		} else {
-			detail.ResultOutputValues = nil
-		}
-		if detail.CreatedAt == nil || detail.CreatedAt.IsZero() {
-			detail.CreatedAt = clone.CreatedAt
-		}
-		if detail.UpdatedAt == nil || detail.UpdatedAt.IsZero() {
-			detail.UpdatedAt = clone.UpdatedAt
-		}
-		if detail.CompletedAt == nil || detail.CompletedAt.IsZero() {
-			detail.CompletedAt = clone.CompletedAt
-		}
-	})
-
-	r.appendActivity(JobActivity{
-		JobID:      completed.JobId,
-		JobType:    completed.JobType,
-		RequestID:  completed.RequestId,
-		WorkerID:   clone.WorkerID,
-		Source:     "worker_completion",
-		Message:    clone.Message,
-		Stage:      clone.Stage,
-		OccurredAt: timeToPtr(now),
-	})
-
-	return &clone
-}
-
-func (r *Plugin) trackWorkerActivities(jobType, jobID, requestID, workerID string, events []*plugin_pb.ActivityEvent) {
-	if len(events) == 0 {
-		return
-	}
-	for _, event := range events {
-		if event == nil {
-			continue
-		}
-		timestamp := time.Now().UTC()
-		if event.CreatedAt != nil {
-			timestamp = event.CreatedAt.AsTime().UTC()
-		}
-		r.appendActivity(JobActivity{
-			JobID:      jobID,
-			JobType:    jobType,
-			RequestID:  requestID,
-			WorkerID:   workerID,
-			Source:     strings.ToLower(event.Source.String()),
-			Message:    event.Message,
-			Stage:      event.Stage,
-			Details:    configValueMapToPlain(event.Details),
-			OccurredAt: timeToPtr(timestamp),
-		})
-	}
-}
-
-func (r *Plugin) appendActivity(activity JobActivity) {
-	if activity.OccurredAt == nil || activity.OccurredAt.IsZero() {
-		activity.OccurredAt = timeToPtr(time.Now().UTC())
-	}
-
-	r.activitiesMu.Lock()
-	r.activities = append(r.activities, activity)
-	if len(r.activities) > maxActivityRecords {
-		r.activities = r.activities[len(r.activities)-maxActivityRecords:]
-	}
-	r.dirtyActivities = true
-	r.activitiesMu.Unlock()
-}
-
-func (r *Plugin) pruneTrackedJobsLocked() {
-	if len(r.jobs) <= maxTrackedJobsTotal {
-		return
-	}
-
-	type sortableJob struct {
-		jobID     string
-		updatedAt time.Time
-	}
-	terminalJobs := make([]sortableJob, 0)
-	for jobID, job := range r.jobs {
-		if job.State == StateSucceeded ||
-			job.State == StateFailed ||
-			job.State == StateCanceled {
-			updAt := time.Time{}
-			if job.UpdatedAt != nil {
-				updAt = *job.UpdatedAt
-			}
-			terminalJobs = append(terminalJobs, sortableJob{jobID, updAt})
-		}
-	}
-
-	if len(terminalJobs) == 0 {
-		return
-	}
-
-	sort.Slice(terminalJobs, func(i, j int) bool {
-		return terminalJobs[i].updatedAt.Before(terminalJobs[j].updatedAt)
-	})
-
-	toDelete := len(r.jobs) - maxTrackedJobsTotal
-	if toDelete <= 0 {
-		return
-	}
-	if toDelete > len(terminalJobs) {
-		toDelete = len(terminalJobs)
-	}
-
-	for i := 0; i < toDelete; i++ {
-		delete(r.jobs, terminalJobs[i].jobID)
-	}
-}
-
-func configValueMapToPlain(values map[string]*plugin_pb.ConfigValue) map[string]interface{} {
-	if len(values) == 0 {
-		return nil
-	}
-
-	payload, err := protojson.MarshalOptions{UseProtoNames: true}.Marshal(&plugin_pb.ValueMap{Fields: values})
-	if err != nil {
-		return nil
-	}
-
-	decoded := map[string]interface{}{}
-	if err := json.Unmarshal(payload, &decoded); err != nil {
-		return nil
-	}
-
-	fields, ok := decoded["fields"].(map[string]interface{})
-	if !ok {
-		return nil
-	}
-	return fields
-}
-
-func (r *Plugin) persistTrackedJobsSnapshot() {
-	r.jobsMu.Lock()
-	r.dirtyJobs = false
-	jobs := make([]TrackedJob, 0, len(r.jobs))
-	for _, job := range r.jobs {
-		if job == nil || strings.TrimSpace(job.JobID) == "" {
-			continue
-		}
-		clone := cloneTrackedJob(*job)
-		stripTrackedJobDetailFields(&clone)
-		jobs = append(jobs, clone)
-	}
-	r.jobsMu.Unlock()
-
-	if len(jobs) == 0 {
-		return
-	}
-
-	sort.Slice(jobs, func(i, j int) bool {
-		ti := time.Time{}
-		if jobs[i].UpdatedAt != nil {
-			ti = *jobs[i].UpdatedAt
-		}
-		tj := time.Time{}
-		if jobs[j].UpdatedAt != nil {
-			tj = *jobs[j].UpdatedAt
-		}
-		if !ti.Equal(tj) {
-			return ti.After(tj)
-		}
-		return jobs[i].JobID < jobs[j].JobID
-	})
-	if len(jobs) > maxTrackedJobsTotal {
-		jobs = jobs[:maxTrackedJobsTotal]
-	}
-
-	if err := r.store.SaveTrackedJobs(jobs); err != nil {
-		glog.Warningf("Plugin failed to persist tracked jobs: %v", err)
-	}
-}
-
-func (r *Plugin) persistJobDetailSnapshot(jobID string, apply func(detail *TrackedJob)) {
-	normalizedJobID, _ := sanitizeJobID(jobID)
-	if normalizedJobID == "" {
-		return
-	}
-
-	r.jobDetailsMu.Lock()
-	defer r.jobDetailsMu.Unlock()
-
-	detail, err := r.store.LoadJobDetail(normalizedJobID)
-	if err != nil {
-		glog.Warningf("Plugin failed to load job detail snapshot for %s: %v", normalizedJobID, err)
-		return
-	}
-	if detail == nil {
-		detail = &TrackedJob{
-			JobID: normalizedJobID,
-		}
-	}
-
-	if apply != nil {
-		apply(detail)
-	}
-
-	if err := r.store.SaveJobDetail(*detail); err != nil {
-		glog.Warningf("Plugin failed to persist job detail snapshot for %s: %v", normalizedJobID, err)
-	}
-}
-
-func (r *Plugin) persistActivitiesSnapshot() {
-	r.activitiesMu.Lock()
-	r.dirtyActivities = false
-	activities := append([]JobActivity(nil), r.activities...)
-	r.activitiesMu.Unlock()
-
-	if len(activities) == 0 {
-		return
-	}
-
-	if len(activities) > maxActivityRecords {
-		activities = activities[len(activities)-maxActivityRecords:]
-	}
-
-	if err := r.store.SaveActivities(activities); err != nil {
-		glog.Warningf("Plugin failed to persist activities: %v", err)
-	}
-}
-
-func (r *Plugin) persistenceLoop() {
-	defer r.wg.Done()
-	for {
-		select {
-		case <-r.shutdownCh:
-			r.persistTrackedJobsSnapshot()
-			r.persistActivitiesSnapshot()
-			return
-		case <-r.persistTicker.C:
-			r.jobsMu.RLock()
-			needsJobsFlush := r.dirtyJobs
-			r.jobsMu.RUnlock()
-			if needsJobsFlush {
-				r.persistTrackedJobsSnapshot()
-			}
-
-			r.activitiesMu.RLock()
-			needsActivitiesFlush := r.dirtyActivities
-			r.activitiesMu.RUnlock()
-			if needsActivitiesFlush {
-				r.persistActivitiesSnapshot()
-			}
-		}
-	}
-}

weed/admin/plugin/plugin_monitor_test.go

@@ -1,600 +0,0 @@
-package plugin
-
-import (
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/timestamppb"
-)
-
-func TestPluginLoadsPersistedMonitorStateOnStart(t *testing.T) {
-	t.Parallel()
-
-	dataDir := t.TempDir()
-	store, err := NewConfigStore(dataDir)
-	if err != nil {
-		t.Fatalf("NewConfigStore: %v", err)
-	}
-
-	seedJobs := []TrackedJob{
-		{
-			JobID:     "job-seeded",
-			JobType:   "vacuum",
-			State:     "running",
-			CreatedAt: timeToPtr(time.Now().UTC().Add(-2 * time.Minute)),
-			UpdatedAt: timeToPtr(time.Now().UTC().Add(-1 * time.Minute)),
-		},
-	}
-	seedActivities := []JobActivity{
-		{
-			JobID:      "job-seeded",
-			JobType:    "vacuum",
-			Source:     "worker_progress",
-			Message:    "seeded",
-			OccurredAt: timeToPtr(time.Now().UTC().Add(-30 * time.Second)),
-		},
-	}
-
-	if err := store.SaveTrackedJobs(seedJobs); err != nil {
-		t.Fatalf("SaveTrackedJobs: %v", err)
-	}
-	if err := store.SaveActivities(seedActivities); err != nil {
-		t.Fatalf("SaveActivities: %v", err)
-	}
-
-	pluginSvc, err := New(Options{DataDir: dataDir})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	gotJobs := pluginSvc.ListTrackedJobs("", "", 0)
-	if len(gotJobs) != 1 || gotJobs[0].JobID != "job-seeded" {
-		t.Fatalf("unexpected loaded jobs: %+v", gotJobs)
-	}
-
-	gotActivities := pluginSvc.ListActivities("", 0)
-	if len(gotActivities) != 1 || gotActivities[0].Message != "seeded" {
-		t.Fatalf("unexpected loaded activities: %+v", gotActivities)
-	}
-}
-
-func TestPluginPersistsMonitorStateAfterJobUpdates(t *testing.T) {
-	t.Parallel()
-
-	dataDir := t.TempDir()
-	pluginSvc, err := New(Options{DataDir: dataDir})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	job := &plugin_pb.JobSpec{
-		JobId:   "job-persist",
-		JobType: "vacuum",
-		Summary: "persist test",
-	}
-	pluginSvc.trackExecutionStart("req-persist", "worker-a", job, 1)
-
-	pluginSvc.trackExecutionCompletion(&plugin_pb.JobCompleted{
-		RequestId:   "req-persist",
-		JobId:       "job-persist",
-		JobType:     "vacuum",
-		Success:     true,
-		Result:      &plugin_pb.JobResult{Summary: "done"},
-		CompletedAt: timestamppb.New(time.Now().UTC()),
-	})
-	pluginSvc.Shutdown()
-
-	store, err := NewConfigStore(dataDir)
-	if err != nil {
-		t.Fatalf("NewConfigStore: %v", err)
-	}
-
-	trackedJobs, err := store.LoadTrackedJobs()
-	if err != nil {
-		t.Fatalf("LoadTrackedJobs: %v", err)
-	}
-	if len(trackedJobs) == 0 {
-		t.Fatalf("expected persisted tracked jobs")
-	}
-
-	found := false
-	for _, tracked := range trackedJobs {
-		if tracked.JobID == "job-persist" {
-			found = true
-			if tracked.State == "" {
-				t.Fatalf("persisted job state should not be empty")
-			}
-		}
-	}
-	if !found {
-		t.Fatalf("persisted tracked jobs missing job-persist")
-	}
-
-	activities, err := store.LoadActivities()
-	if err != nil {
-		t.Fatalf("LoadActivities: %v", err)
-	}
-	if len(activities) == 0 {
-		t.Fatalf("expected persisted activities")
-	}
-}
-
-func TestTrackExecutionQueuedMarksPendingState(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	pluginSvc.trackExecutionQueued(&plugin_pb.JobSpec{
-		JobId:     "job-pending-1",
-		JobType:   "vacuum",
-		DedupeKey: "vacuum:1",
-		Summary:   "pending queue item",
-	})
-
-	jobs := pluginSvc.ListTrackedJobs("vacuum", "", 10)
-	if len(jobs) != 1 {
-		t.Fatalf("expected one tracked pending job, got=%d", len(jobs))
-	}
-	job := jobs[0]
-	if job.JobID != "job-pending-1" {
-		t.Fatalf("unexpected pending job id: %s", job.JobID)
-	}
-	if job.State != "job_state_pending" {
-		t.Fatalf("unexpected pending job state: %s", job.State)
-	}
-	if job.Stage != "queued" {
-		t.Fatalf("unexpected pending job stage: %s", job.Stage)
-	}
-
-	activities := pluginSvc.ListActivities("vacuum", 50)
-	found := false
-	for _, activity := range activities {
-		if activity.JobID == "job-pending-1" && activity.Stage == "queued" && activity.Source == "admin_scheduler" {
-			found = true
-			break
-		}
-	}
-	if !found {
-		t.Fatalf("expected queued activity for pending job")
-	}
-}
-
-func TestHandleJobProgressUpdateCarriesWorkerIDInActivities(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	job := &plugin_pb.JobSpec{
-		JobId:   "job-progress-worker",
-		JobType: "vacuum",
-	}
-	pluginSvc.trackExecutionStart("req-progress-worker", "worker-a", job, 1)
-
-	pluginSvc.handleJobProgressUpdate("worker-a", &plugin_pb.JobProgressUpdate{
-		RequestId:       "req-progress-worker",
-		JobId:           "job-progress-worker",
-		JobType:         "vacuum",
-		State:           plugin_pb.JobState_JOB_STATE_RUNNING,
-		ProgressPercent: 42.0,
-		Stage:           "scan",
-		Message:         "in progress",
-		Activities: []*plugin_pb.ActivityEvent{
-			{
-				Source:  plugin_pb.ActivitySource_ACTIVITY_SOURCE_EXECUTOR,
-				Message: "volume scanned",
-				Stage:   "scan",
-			},
-		},
-	})
-
-	activities := pluginSvc.ListActivities("vacuum", 0)
-	if len(activities) == 0 {
-		t.Fatalf("expected activity entries")
-	}
-
-	foundProgress := false
-	foundEvent := false
-	for _, activity := range activities {
-		if activity.Source == "worker_progress" && activity.Message == "in progress" {
-			foundProgress = true
-			if activity.WorkerID != "worker-a" {
-				t.Fatalf("worker_progress activity worker mismatch: got=%q want=%q", activity.WorkerID, "worker-a")
-			}
-		}
-		if activity.Message == "volume scanned" {
-			foundEvent = true
-			if activity.WorkerID != "worker-a" {
-				t.Fatalf("worker event worker mismatch: got=%q want=%q", activity.WorkerID, "worker-a")
-			}
-		}
-	}
-
-	if !foundProgress {
-		t.Fatalf("expected worker_progress activity")
-	}
-	if !foundEvent {
-		t.Fatalf("expected worker activity event")
-	}
-}
-
-func TestHandleJobProgressUpdateWithoutJobIDTracksDetectionActivities(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	pluginSvc.handleJobProgressUpdate("worker-detector", &plugin_pb.JobProgressUpdate{
-		RequestId: "detect-req-1",
-		JobType:   "vacuum",
-		State:     plugin_pb.JobState_JOB_STATE_RUNNING,
-		Stage:     "decision_summary",
-		Message:   "VACUUM: No tasks created for 3 volumes",
-		Activities: []*plugin_pb.ActivityEvent{
-			{
-				Source:  plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR,
-				Stage:   "decision_summary",
-				Message: "VACUUM: No tasks created for 3 volumes",
-			},
-		},
-	})
-
-	activities := pluginSvc.ListActivities("vacuum", 0)
-	if len(activities) == 0 {
-		t.Fatalf("expected activity entries")
-	}
-
-	foundDetectionProgress := false
-	foundDetectorEvent := false
-	for _, activity := range activities {
-		if activity.RequestID != "detect-req-1" {
-			continue
-		}
-		if activity.Source == "worker_detection" {
-			foundDetectionProgress = true
-			if activity.WorkerID != "worker-detector" {
-				t.Fatalf("worker_detection worker mismatch: got=%q want=%q", activity.WorkerID, "worker-detector")
-			}
-		}
-		if activity.Source == "activity_source_detector" {
-			foundDetectorEvent = true
-			if activity.WorkerID != "worker-detector" {
-				t.Fatalf("detector event worker mismatch: got=%q want=%q", activity.WorkerID, "worker-detector")
-			}
-		}
-	}
-
-	if !foundDetectionProgress {
-		t.Fatalf("expected worker_detection activity")
-	}
-	if !foundDetectorEvent {
-		t.Fatalf("expected detector activity event")
-	}
-}
-
-func TestHandleJobCompletedCarriesWorkerIDInActivitiesAndRunHistory(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	job := &plugin_pb.JobSpec{
-		JobId:   "job-complete-worker",
-		JobType: "vacuum",
-	}
-	pluginSvc.trackExecutionStart("req-complete-worker", "worker-b", job, 1)
-
-	pluginSvc.handleJobCompleted(&plugin_pb.JobCompleted{
-		RequestId: "req-complete-worker",
-		JobId:     "job-complete-worker",
-		JobType:   "vacuum",
-		Success:   true,
-		Activities: []*plugin_pb.ActivityEvent{
-			{
-				Source:  plugin_pb.ActivitySource_ACTIVITY_SOURCE_EXECUTOR,
-				Message: "finalizer done",
-				Stage:   "finalize",
-			},
-		},
-		CompletedAt: timestamppb.Now(),
-	})
-	pluginSvc.Shutdown()
-
-	activities := pluginSvc.ListActivities("vacuum", 0)
-	foundWorkerEvent := false
-	for _, activity := range activities {
-		if activity.Message == "finalizer done" {
-			foundWorkerEvent = true
-			if activity.WorkerID != "worker-b" {
-				t.Fatalf("worker completion event worker mismatch: got=%q want=%q", activity.WorkerID, "worker-b")
-			}
-		}
-	}
-	if !foundWorkerEvent {
-		t.Fatalf("expected completion worker event activity")
-	}
-
-	history, err := pluginSvc.LoadRunHistory("vacuum")
-	if err != nil {
-		t.Fatalf("LoadRunHistory: %v", err)
-	}
-	if history == nil || len(history.SuccessfulRuns) == 0 {
-		t.Fatalf("expected successful run history entry")
-	}
-	if history.SuccessfulRuns[0].WorkerID != "worker-b" {
-		t.Fatalf("run history worker mismatch: got=%q want=%q", history.SuccessfulRuns[0].WorkerID, "worker-b")
-	}
-}
-
-func TestTrackExecutionStartStoresJobPayloadDetails(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{DataDir: t.TempDir()})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	pluginSvc.trackExecutionStart("req-payload", "worker-c", &plugin_pb.JobSpec{
-		JobId:   "job-payload",
-		JobType: "vacuum",
-		Summary: "payload summary",
-		Detail:  "payload detail",
-		Parameters: map[string]*plugin_pb.ConfigValue{
-			"volume_id": {
-				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 9},
-			},
-		},
-		Labels: map[string]string{
-			"source": "detector",
-		},
-	}, 2)
-	pluginSvc.Shutdown()
-
-	job, found := pluginSvc.GetTrackedJob("job-payload")
-	if !found || job == nil {
-		t.Fatalf("expected tracked job")
-	}
-	if job.Detail != "" {
-		t.Fatalf("expected in-memory tracked job detail to be stripped, got=%q", job.Detail)
-	}
-	if job.Attempt != 2 {
-		t.Fatalf("unexpected attempt: %d", job.Attempt)
-	}
-	if len(job.Labels) != 0 {
-		t.Fatalf("expected in-memory labels to be stripped, got=%+v", job.Labels)
-	}
-	if len(job.Parameters) != 0 {
-		t.Fatalf("expected in-memory parameters to be stripped, got=%+v", job.Parameters)
-	}
-
-	detail, found, err := pluginSvc.BuildJobDetail("job-payload", 100, 0)
-	if err != nil {
-		t.Fatalf("BuildJobDetail: %v", err)
-	}
-	if !found || detail == nil || detail.Job == nil {
-		t.Fatalf("expected disk-backed job detail")
-	}
-	if detail.Job.Detail != "payload detail" {
-		t.Fatalf("unexpected disk-backed detail: %q", detail.Job.Detail)
-	}
-	if got := detail.Job.Labels["source"]; got != "detector" {
-		t.Fatalf("unexpected disk-backed label source: %q", got)
-	}
-	if got, ok := detail.Job.Parameters["volume_id"].(map[string]interface{}); !ok || got["int64_value"] != "9" {
-		t.Fatalf("unexpected disk-backed parameters payload: %#v", detail.Job.Parameters["volume_id"])
-	}
-}
-
-func TestTrackExecutionStartStoresErasureCodingExecutionPlan(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{DataDir: t.TempDir()})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	taskParams := &worker_pb.TaskParams{
-		TaskId:     "task-ec-1",
-		VolumeId:   29,
-		Collection: "photos",
-		Sources: []*worker_pb.TaskSource{
-			{
-				Node:       "source-a:8080",
-				DataCenter: "dc1",
-				Rack:       "rack1",
-				VolumeId:   29,
-			},
-		},
-		Targets: []*worker_pb.TaskTarget{
-			{
-				Node:       "target-a:8080",
-				DataCenter: "dc1",
-				Rack:       "rack2",
-				VolumeId:   29,
-				ShardIds:   []uint32{0, 10},
-			},
-			{
-				Node:       "target-b:8080",
-				DataCenter: "dc2",
-				Rack:       "rack3",
-				VolumeId:   29,
-				ShardIds:   []uint32{1, 11},
-			},
-		},
-		TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
-			ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
-				DataShards:   10,
-				ParityShards: 4,
-			},
-		},
-	}
-	payload, err := proto.Marshal(taskParams)
-	if err != nil {
-		t.Fatalf("Marshal task params: %v", err)
-	}
-
-	pluginSvc.trackExecutionStart("req-ec-plan", "worker-ec", &plugin_pb.JobSpec{
-		JobId:   "job-ec-plan",
-		JobType: "erasure_coding",
-		Parameters: map[string]*plugin_pb.ConfigValue{
-			"task_params_pb": {
-				Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload},
-			},
-		},
-	}, 1)
-	pluginSvc.Shutdown()
-
-	detail, found, err := pluginSvc.BuildJobDetail("job-ec-plan", 100, 0)
-	if err != nil {
-		t.Fatalf("BuildJobDetail: %v", err)
-	}
-	if !found || detail == nil || detail.Job == nil {
-		t.Fatalf("expected disk-backed detail")
-	}
-
-	rawPlan, ok := detail.Job.Parameters["execution_plan"]
-	if !ok {
-		t.Fatalf("expected execution_plan in parameters, got=%+v", detail.Job.Parameters)
-	}
-	plan, ok := rawPlan.(map[string]interface{})
-	if !ok {
-		t.Fatalf("unexpected execution_plan type: %T", rawPlan)
-	}
-	if plan["job_type"] != "erasure_coding" {
-		t.Fatalf("unexpected execution plan job type: %+v", plan["job_type"])
-	}
-	if plan["volume_id"] != float64(29) {
-		t.Fatalf("unexpected execution plan volume id: %+v", plan["volume_id"])
-	}
-	targets, ok := plan["targets"].([]interface{})
-	if !ok || len(targets) != 2 {
-		t.Fatalf("unexpected targets in execution plan: %+v", plan["targets"])
-	}
-	assignments, ok := plan["shard_assignments"].([]interface{})
-	if !ok || len(assignments) != 4 {
-		t.Fatalf("unexpected shard assignments in execution plan: %+v", plan["shard_assignments"])
-	}
-	firstAssignment, ok := assignments[0].(map[string]interface{})
-	if !ok {
-		t.Fatalf("unexpected first assignment payload: %+v", assignments[0])
-	}
-	if firstAssignment["shard_id"] != float64(0) || firstAssignment["kind"] != "data" {
-		t.Fatalf("unexpected first assignment: %+v", firstAssignment)
-	}
-}
-
-func TestBuildJobDetailIncludesActivitiesAndRunRecord(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{DataDir: t.TempDir()})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	pluginSvc.trackExecutionStart("req-detail", "worker-z", &plugin_pb.JobSpec{
-		JobId:   "job-detail",
-		JobType: "vacuum",
-		Summary: "detail summary",
-	}, 1)
-	pluginSvc.handleJobProgressUpdate("worker-z", &plugin_pb.JobProgressUpdate{
-		RequestId: "req-detail",
-		JobId:     "job-detail",
-		JobType:   "vacuum",
-		State:     plugin_pb.JobState_JOB_STATE_RUNNING,
-		Stage:     "scan",
-		Message:   "scanning volume",
-	})
-	pluginSvc.handleJobCompleted(&plugin_pb.JobCompleted{
-		RequestId: "req-detail",
-		JobId:     "job-detail",
-		JobType:   "vacuum",
-		Success:   true,
-		Result: &plugin_pb.JobResult{
-			Summary: "done",
-			OutputValues: map[string]*plugin_pb.ConfigValue{
-				"affected": {
-					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1},
-				},
-			},
-		},
-		CompletedAt: timestamppb.Now(),
-	})
-	pluginSvc.Shutdown()
-
-	detail, found, err := pluginSvc.BuildJobDetail("job-detail", 100, 5)
-	if err != nil {
-		t.Fatalf("BuildJobDetail error: %v", err)
-	}
-	if !found || detail == nil {
-		t.Fatalf("expected job detail")
-	}
-	if detail.Job == nil || detail.Job.JobID != "job-detail" {
-		t.Fatalf("unexpected job detail payload: %+v", detail.Job)
-	}
-	if detail.RunRecord == nil || detail.RunRecord.JobID != "job-detail" {
-		t.Fatalf("expected run record for job-detail, got=%+v", detail.RunRecord)
-	}
-	if len(detail.Activities) == 0 {
-		t.Fatalf("expected activity timeline entries")
-	}
-	if detail.Job.ResultOutputValues == nil {
-		t.Fatalf("expected result output values")
-	}
-}
-
-func TestBuildJobDetailLoadsFromDiskWhenMemoryCleared(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{DataDir: t.TempDir()})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	pluginSvc.trackExecutionStart("req-disk", "worker-d", &plugin_pb.JobSpec{
-		JobId:   "job-disk",
-		JobType: "vacuum",
-		Summary: "disk summary",
-		Detail:  "disk detail payload",
-	}, 1)
-	pluginSvc.Shutdown()
-
-	pluginSvc.jobsMu.Lock()
-	pluginSvc.jobs = map[string]*TrackedJob{}
-	pluginSvc.jobsMu.Unlock()
-	pluginSvc.activitiesMu.Lock()
-	pluginSvc.activities = nil
-	pluginSvc.activitiesMu.Unlock()
-
-	detail, found, err := pluginSvc.BuildJobDetail("job-disk", 100, 0)
-	if err != nil {
-		t.Fatalf("BuildJobDetail: %v", err)
-	}
-	if !found || detail == nil || detail.Job == nil {
-		t.Fatalf("expected detail from disk")
-	}
-	if detail.Job.Detail != "disk detail payload" {
-		t.Fatalf("unexpected disk detail payload: %q", detail.Job.Detail)
-	}
-}

weed/admin/plugin/plugin_scheduler.go

@@ -1,945 +0,0 @@
-package plugin
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-	"google.golang.org/protobuf/types/known/timestamppb"
-)
-
-var errExecutorAtCapacity = errors.New("executor is at capacity")
-
-const (
-	defaultSchedulerTick                       = 5 * time.Second
-	defaultScheduledDetectionInterval          = 300 * time.Second
-	defaultScheduledDetectionTimeout           = 45 * time.Second
-	defaultScheduledExecutionTimeout           = 90 * time.Second
-	defaultScheduledMaxResults           int32 = 1000
-	defaultScheduledExecutionConcurrency       = 1
-	defaultScheduledPerWorkerConcurrency       = 1
-	maxScheduledExecutionConcurrency           = 128
-	defaultScheduledRetryBackoff               = 5 * time.Second
-	defaultClusterContextTimeout               = 10 * time.Second
-	defaultWaitingBacklogFloor                 = 8
-	defaultWaitingBacklogMultiplier            = 4
-)
-
-type schedulerPolicy struct {
-	DetectionInterval      time.Duration
-	DetectionTimeout       time.Duration
-	ExecutionTimeout       time.Duration
-	RetryBackoff           time.Duration
-	MaxResults             int32
-	ExecutionConcurrency   int
-	PerWorkerConcurrency   int
-	RetryLimit             int
-	ExecutorReserveBackoff time.Duration
-}
-
-func (r *Plugin) schedulerLoop() {
-	defer r.wg.Done()
-	ticker := time.NewTicker(r.schedulerTick)
-	defer ticker.Stop()
-
-	// Try once immediately on startup.
-	r.runSchedulerTick()
-
-	for {
-		select {
-		case <-r.shutdownCh:
-			return
-		case <-ticker.C:
-			r.runSchedulerTick()
-		}
-	}
-}
-
-func (r *Plugin) runSchedulerTick() {
-	jobTypes := r.registry.DetectableJobTypes()
-	if len(jobTypes) == 0 {
-		return
-	}
-
-	active := make(map[string]struct{}, len(jobTypes))
-	for _, jobType := range jobTypes {
-		active[jobType] = struct{}{}
-
-		policy, enabled, err := r.loadSchedulerPolicy(jobType)
-		if err != nil {
-			glog.Warningf("Plugin scheduler failed to load policy for %s: %v", jobType, err)
-			continue
-		}
-		if !enabled {
-			r.clearSchedulerJobType(jobType)
-			continue
-		}
-
-		if !r.markDetectionDue(jobType, policy.DetectionInterval) {
-			continue
-		}
-
-		r.wg.Add(1)
-		go func(jt string, p schedulerPolicy) {
-			defer r.wg.Done()
-			r.runScheduledDetection(jt, p)
-		}(jobType, policy)
-	}
-
-	r.pruneSchedulerState(active)
-	r.pruneDetectorLeases(active)
-}
-
-func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, error) {
-	cfg, err := r.store.LoadJobTypeConfig(jobType)
-	if err != nil {
-		return schedulerPolicy{}, false, err
-	}
-	descriptor, err := r.store.LoadDescriptor(jobType)
-	if err != nil {
-		return schedulerPolicy{}, false, err
-	}
-
-	adminRuntime := deriveSchedulerAdminRuntime(cfg, descriptor)
-	if adminRuntime == nil {
-		return schedulerPolicy{}, false, nil
-	}
-	if !adminRuntime.Enabled {
-		return schedulerPolicy{}, false, nil
-	}
-
-	policy := schedulerPolicy{
-		DetectionInterval:      durationFromSeconds(adminRuntime.DetectionIntervalSeconds, defaultScheduledDetectionInterval),
-		DetectionTimeout:       durationFromSeconds(adminRuntime.DetectionTimeoutSeconds, defaultScheduledDetectionTimeout),
-		ExecutionTimeout:       defaultScheduledExecutionTimeout,
-		RetryBackoff:           durationFromSeconds(adminRuntime.RetryBackoffSeconds, defaultScheduledRetryBackoff),
-		MaxResults:             adminRuntime.MaxJobsPerDetection,
-		ExecutionConcurrency:   int(adminRuntime.GlobalExecutionConcurrency),
-		PerWorkerConcurrency:   int(adminRuntime.PerWorkerExecutionConcurrency),
-		RetryLimit:             int(adminRuntime.RetryLimit),
-		ExecutorReserveBackoff: 200 * time.Millisecond,
-	}
-
-	if policy.DetectionInterval < r.schedulerTick {
-		policy.DetectionInterval = r.schedulerTick
-	}
-	if policy.MaxResults <= 0 {
-		policy.MaxResults = defaultScheduledMaxResults
-	}
-	if policy.ExecutionConcurrency <= 0 {
-		policy.ExecutionConcurrency = defaultScheduledExecutionConcurrency
-	}
-	if policy.ExecutionConcurrency > maxScheduledExecutionConcurrency {
-		policy.ExecutionConcurrency = maxScheduledExecutionConcurrency
-	}
-	if policy.PerWorkerConcurrency <= 0 {
-		policy.PerWorkerConcurrency = defaultScheduledPerWorkerConcurrency
-	}
-	if policy.PerWorkerConcurrency > policy.ExecutionConcurrency {
-		policy.PerWorkerConcurrency = policy.ExecutionConcurrency
-	}
-	if policy.RetryLimit < 0 {
-		policy.RetryLimit = 0
-	}
-
-	// Plugin protocol currently has only detection timeout in admin settings.
-	execTimeout := time.Duration(adminRuntime.DetectionTimeoutSeconds*2) * time.Second
-	if execTimeout < defaultScheduledExecutionTimeout {
-		execTimeout = defaultScheduledExecutionTimeout
-	}
-	policy.ExecutionTimeout = execTimeout
-
-	return policy, true, nil
-}
-
-func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) {
-	jobTypes, err := r.ListKnownJobTypes()
-	if err != nil {
-		return nil, err
-	}
-
-	r.schedulerMu.Lock()
-	nextDetectionAt := make(map[string]time.Time, len(r.nextDetectionAt))
-	for jobType, nextRun := range r.nextDetectionAt {
-		nextDetectionAt[jobType] = nextRun
-	}
-	detectionInFlight := make(map[string]bool, len(r.detectionInFlight))
-	for jobType, inFlight := range r.detectionInFlight {
-		detectionInFlight[jobType] = inFlight
-	}
-	r.schedulerMu.Unlock()
-
-	states := make([]SchedulerJobTypeState, 0, len(jobTypes))
-	for _, jobType := range jobTypes {
-		state := SchedulerJobTypeState{
-			JobType:           jobType,
-			DetectionInFlight: detectionInFlight[jobType],
-		}
-
-		if nextRun, ok := nextDetectionAt[jobType]; ok && !nextRun.IsZero() {
-			nextRunUTC := nextRun.UTC()
-			state.NextDetectionAt = &nextRunUTC
-		}
-
-		policy, enabled, loadErr := r.loadSchedulerPolicy(jobType)
-		if loadErr != nil {
-			state.PolicyError = loadErr.Error()
-		} else {
-			state.Enabled = enabled
-			if enabled {
-				state.DetectionIntervalSeconds = secondsFromDuration(policy.DetectionInterval)
-				state.DetectionTimeoutSeconds = secondsFromDuration(policy.DetectionTimeout)
-				state.ExecutionTimeoutSeconds = secondsFromDuration(policy.ExecutionTimeout)
-				state.MaxJobsPerDetection = policy.MaxResults
-				state.GlobalExecutionConcurrency = policy.ExecutionConcurrency
-				state.PerWorkerExecutionConcurrency = policy.PerWorkerConcurrency
-				state.RetryLimit = policy.RetryLimit
-				state.RetryBackoffSeconds = secondsFromDuration(policy.RetryBackoff)
-			}
-		}
-
-		leasedWorkerID := r.getDetectorLease(jobType)
-		if leasedWorkerID != "" {
-			state.DetectorWorkerID = leasedWorkerID
-			if worker, ok := r.registry.Get(leasedWorkerID); ok {
-				if capability := worker.Capabilities[jobType]; capability != nil && capability.CanDetect {
-					state.DetectorAvailable = true
-				}
-			}
-		}
-		if state.DetectorWorkerID == "" {
-			detector, detectorErr := r.registry.PickDetector(jobType)
-			if detectorErr == nil && detector != nil {
-				state.DetectorAvailable = true
-				state.DetectorWorkerID = detector.WorkerID
-			}
-		}
-
-		executors, executorErr := r.registry.ListExecutors(jobType)
-		if executorErr == nil {
-			state.ExecutorWorkerCount = len(executors)
-		}
-
-		states = append(states, state)
-	}
-
-	return states, nil
-}
-
-func deriveSchedulerAdminRuntime(
-	cfg *plugin_pb.PersistedJobTypeConfig,
-	descriptor *plugin_pb.JobTypeDescriptor,
-) *plugin_pb.AdminRuntimeConfig {
-	if cfg != nil && cfg.AdminRuntime != nil {
-		adminConfig := *cfg.AdminRuntime
-		return &adminConfig
-	}
-
-	if descriptor == nil || descriptor.AdminRuntimeDefaults == nil {
-		return nil
-	}
-
-	defaults := descriptor.AdminRuntimeDefaults
-	return &plugin_pb.AdminRuntimeConfig{
-		Enabled:                       defaults.Enabled,
-		DetectionIntervalSeconds:      defaults.DetectionIntervalSeconds,
-		DetectionTimeoutSeconds:       defaults.DetectionTimeoutSeconds,
-		MaxJobsPerDetection:           defaults.MaxJobsPerDetection,
-		GlobalExecutionConcurrency:    defaults.GlobalExecutionConcurrency,
-		PerWorkerExecutionConcurrency: defaults.PerWorkerExecutionConcurrency,
-		RetryLimit:                    defaults.RetryLimit,
-		RetryBackoffSeconds:           defaults.RetryBackoffSeconds,
-	}
-}
-
-func (r *Plugin) markDetectionDue(jobType string, interval time.Duration) bool {
-	now := time.Now().UTC()
-
-	r.schedulerMu.Lock()
-	defer r.schedulerMu.Unlock()
-
-	if r.detectionInFlight[jobType] {
-		return false
-	}
-
-	nextRun, exists := r.nextDetectionAt[jobType]
-	if exists && now.Before(nextRun) {
-		return false
-	}
-
-	r.nextDetectionAt[jobType] = now.Add(interval)
-	r.detectionInFlight[jobType] = true
-	return true
-}
-
-func (r *Plugin) finishDetection(jobType string) {
-	r.schedulerMu.Lock()
-	delete(r.detectionInFlight, jobType)
-	r.schedulerMu.Unlock()
-}
-
-func (r *Plugin) pruneSchedulerState(activeJobTypes map[string]struct{}) {
-	r.schedulerMu.Lock()
-	defer r.schedulerMu.Unlock()
-
-	for jobType := range r.nextDetectionAt {
-		if _, ok := activeJobTypes[jobType]; !ok {
-			delete(r.nextDetectionAt, jobType)
-			delete(r.detectionInFlight, jobType)
-		}
-	}
-}
-
-func (r *Plugin) clearSchedulerJobType(jobType string) {
-	r.schedulerMu.Lock()
-	delete(r.nextDetectionAt, jobType)
-	delete(r.detectionInFlight, jobType)
-	r.schedulerMu.Unlock()
-	r.clearDetectorLease(jobType, "")
-}
-
-func (r *Plugin) pruneDetectorLeases(activeJobTypes map[string]struct{}) {
-	r.detectorLeaseMu.Lock()
-	defer r.detectorLeaseMu.Unlock()
-
-	for jobType := range r.detectorLeases {
-		if _, ok := activeJobTypes[jobType]; !ok {
-			delete(r.detectorLeases, jobType)
-		}
-	}
-}
-
-func (r *Plugin) runScheduledDetection(jobType string, policy schedulerPolicy) {
-	defer r.finishDetection(jobType)
-
-	start := time.Now().UTC()
-	r.appendActivity(JobActivity{
-		JobType:    jobType,
-		Source:     "admin_scheduler",
-		Message:    "scheduled detection started",
-		Stage:      "detecting",
-		OccurredAt: timeToPtr(start),
-	})
-
-	if skip, waitingCount, waitingThreshold := r.shouldSkipDetectionForWaitingJobs(jobType, policy); skip {
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection skipped: waiting backlog %d reached threshold %d", waitingCount, waitingThreshold),
-			Stage:      "skipped_waiting_backlog",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-		return
-	}
-
-	clusterContext, err := r.loadSchedulerClusterContext()
-	if err != nil {
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection aborted: %v", err),
-			Stage:      "failed",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-		return
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), policy.DetectionTimeout)
-	proposals, err := r.RunDetection(ctx, jobType, clusterContext, policy.MaxResults)
-	cancel()
-	if err != nil {
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection failed: %v", err),
-			Stage:      "failed",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-		return
-	}
-
-	r.appendActivity(JobActivity{
-		JobType:    jobType,
-		Source:     "admin_scheduler",
-		Message:    fmt.Sprintf("scheduled detection completed: %d proposal(s)", len(proposals)),
-		Stage:      "detected",
-		OccurredAt: timeToPtr(time.Now().UTC()),
-	})
-
-	filteredByActive, skippedActive := r.filterProposalsWithActiveJobs(jobType, proposals)
-	if skippedActive > 0 {
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection skipped %d proposal(s) due to active assigned/running jobs", skippedActive),
-			Stage:      "deduped_active_jobs",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-	}
-
-	if len(filteredByActive) == 0 {
-		return
-	}
-
-	filtered := r.filterScheduledProposals(filteredByActive)
-	if len(filtered) != len(filteredByActive) {
-		r.appendActivity(JobActivity{
-			JobType:    jobType,
-			Source:     "admin_scheduler",
-			Message:    fmt.Sprintf("scheduled detection deduped %d proposal(s) within this run", len(filteredByActive)-len(filtered)),
-			Stage:      "deduped",
-			OccurredAt: timeToPtr(time.Now().UTC()),
-		})
-	}
-
-	if len(filtered) == 0 {
-		return
-	}
-
-	r.dispatchScheduledProposals(jobType, filtered, clusterContext, policy)
-}
-
-func (r *Plugin) loadSchedulerClusterContext() (*plugin_pb.ClusterContext, error) {
-	if r.clusterContextProvider == nil {
-		return nil, fmt.Errorf("cluster context provider is not configured")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), defaultClusterContextTimeout)
-	defer cancel()
-
-	clusterContext, err := r.clusterContextProvider(ctx)
-	if err != nil {
-		return nil, err
-	}
-	if clusterContext == nil {
-		return nil, fmt.Errorf("cluster context provider returned nil")
-	}
-	return clusterContext, nil
-}
-
-func (r *Plugin) dispatchScheduledProposals(
-	jobType string,
-	proposals []*plugin_pb.JobProposal,
-	clusterContext *plugin_pb.ClusterContext,
-	policy schedulerPolicy,
-) {
-	jobQueue := make(chan *plugin_pb.JobSpec, len(proposals))
-	for index, proposal := range proposals {
-		job := buildScheduledJobSpec(jobType, proposal, index)
-		r.trackExecutionQueued(job)
-		select {
-		case <-r.shutdownCh:
-			close(jobQueue)
-			return
-		default:
-			jobQueue <- job
-		}
-	}
-	close(jobQueue)
-
-	var wg sync.WaitGroup
-	var statsMu sync.Mutex
-	successCount := 0
-	errorCount := 0
-
-	workerCount := policy.ExecutionConcurrency
-	if workerCount < 1 {
-		workerCount = 1
-	}
-
-	for i := 0; i < workerCount; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
-			for job := range jobQueue {
-				select {
-				case <-r.shutdownCh:
-					return
-				default:
-				}
-
-				for {
-					select {
-					case <-r.shutdownCh:
-						return
-					default:
-					}
-
-					executor, release, reserveErr := r.reserveScheduledExecutor(jobType, policy)
-					if reserveErr != nil {
-						select {
-						case <-r.shutdownCh:
-							return
-						default:
-						}
-						statsMu.Lock()
-						errorCount++
-						statsMu.Unlock()
-						r.appendActivity(JobActivity{
-							JobType:    jobType,
-							Source:     "admin_scheduler",
-							Message:    fmt.Sprintf("scheduled execution reservation failed: %v", reserveErr),
-							Stage:      "failed",
-							OccurredAt: timeToPtr(time.Now().UTC()),
-						})
-						break
-					}
-
-					err := r.executeScheduledJobWithExecutor(executor, job, clusterContext, policy)
-					release()
-					if errors.Is(err, errExecutorAtCapacity) {
-						r.trackExecutionQueued(job)
-						if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
-							return
-						}
-						continue
-					}
-					if err != nil {
-						statsMu.Lock()
-						errorCount++
-						statsMu.Unlock()
-						r.appendActivity(JobActivity{
-							JobID:      job.JobId,
-							JobType:    job.JobType,
-							Source:     "admin_scheduler",
-							Message:    fmt.Sprintf("scheduled execution failed: %v", err),
-							Stage:      "failed",
-							OccurredAt: timeToPtr(time.Now().UTC()),
-						})
-						break
-					}
-
-					statsMu.Lock()
-					successCount++
-					statsMu.Unlock()
-					break
-				}
-			}
-		}()
-	}
-
-	wg.Wait()
-
-	r.appendActivity(JobActivity{
-		JobType:    jobType,
-		Source:     "admin_scheduler",
-		Message:    fmt.Sprintf("scheduled execution finished: success=%d error=%d", successCount, errorCount),
-		Stage:      "executed",
-		OccurredAt: timeToPtr(time.Now().UTC()),
-	})
-}
-
-func (r *Plugin) reserveScheduledExecutor(
-	jobType string,
-	policy schedulerPolicy,
-) (*WorkerSession, func(), error) {
-	deadline := time.Now().Add(policy.ExecutionTimeout)
-	if policy.ExecutionTimeout <= 0 {
-		deadline = time.Now().Add(10 * time.Minute) // Default cap
-	}
-
-	for {
-		select {
-		case <-r.shutdownCh:
-			return nil, nil, fmt.Errorf("plugin is shutting down")
-		default:
-		}
-
-		if time.Now().After(deadline) {
-			return nil, nil, fmt.Errorf("timed out waiting for executor capacity for %s", jobType)
-		}
-
-		executors, err := r.registry.ListExecutors(jobType)
-		if err != nil {
-			if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
-				return nil, nil, fmt.Errorf("plugin is shutting down")
-			}
-			continue
-		}
-
-		for _, executor := range executors {
-			release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy)
-			if !ok {
-				continue
-			}
-			return executor, release, nil
-		}
-
-		if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
-			return nil, nil, fmt.Errorf("plugin is shutting down")
-		}
-	}
-}
-
-func (r *Plugin) tryReserveExecutorCapacity(
-	executor *WorkerSession,
-	jobType string,
-	policy schedulerPolicy,
-) (func(), bool) {
-	if executor == nil || strings.TrimSpace(executor.WorkerID) == "" {
-		return nil, false
-	}
-
-	limit := schedulerWorkerExecutionLimit(executor, jobType, policy)
-	if limit <= 0 {
-		return nil, false
-	}
-	heartbeatUsed := 0
-	if executor.Heartbeat != nil && executor.Heartbeat.ExecutionSlotsUsed > 0 {
-		heartbeatUsed = int(executor.Heartbeat.ExecutionSlotsUsed)
-	}
-
-	workerID := strings.TrimSpace(executor.WorkerID)
-
-	r.schedulerExecMu.Lock()
-	reserved := r.schedulerExecReservations[workerID]
-	if heartbeatUsed+reserved >= limit {
-		r.schedulerExecMu.Unlock()
-		return nil, false
-	}
-	r.schedulerExecReservations[workerID] = reserved + 1
-	r.schedulerExecMu.Unlock()
-
-	release := func() {
-		r.releaseExecutorCapacity(workerID)
-	}
-	return release, true
-}
-
-func (r *Plugin) releaseExecutorCapacity(workerID string) {
-	workerID = strings.TrimSpace(workerID)
-	if workerID == "" {
-		return
-	}
-
-	r.schedulerExecMu.Lock()
-	defer r.schedulerExecMu.Unlock()
-
-	current := r.schedulerExecReservations[workerID]
-	if current <= 1 {
-		delete(r.schedulerExecReservations, workerID)
-		return
-	}
-	r.schedulerExecReservations[workerID] = current - 1
-}
-
-func schedulerWorkerExecutionLimit(executor *WorkerSession, jobType string, policy schedulerPolicy) int {
-	limit := policy.PerWorkerConcurrency
-	if limit <= 0 {
-		limit = defaultScheduledPerWorkerConcurrency
-	}
-
-	if capability := executor.Capabilities[jobType]; capability != nil && capability.MaxExecutionConcurrency > 0 {
-		capLimit := int(capability.MaxExecutionConcurrency)
-		if capLimit < limit {
-			limit = capLimit
-		}
-	}
-
-	if executor.Heartbeat != nil && executor.Heartbeat.ExecutionSlotsTotal > 0 {
-		heartbeatLimit := int(executor.Heartbeat.ExecutionSlotsTotal)
-		if heartbeatLimit < limit {
-			limit = heartbeatLimit
-		}
-	}
-
-	if limit < 0 {
-		return 0
-	}
-	return limit
-}
-
-func (r *Plugin) executeScheduledJobWithExecutor(
-	executor *WorkerSession,
-	job *plugin_pb.JobSpec,
-	clusterContext *plugin_pb.ClusterContext,
-	policy schedulerPolicy,
-) error {
-	maxAttempts := policy.RetryLimit + 1
-	if maxAttempts < 1 {
-		maxAttempts = 1
-	}
-
-	var lastErr error
-	for attempt := 1; attempt <= maxAttempts; attempt++ {
-		select {
-		case <-r.shutdownCh:
-			return fmt.Errorf("plugin is shutting down")
-		default:
-		}
-
-		execCtx, cancel := context.WithTimeout(context.Background(), policy.ExecutionTimeout)
-		_, err := r.executeJobWithExecutor(execCtx, executor, job, clusterContext, int32(attempt))
-		cancel()
-		if err == nil {
-			return nil
-		}
-		if isExecutorAtCapacityError(err) {
-			return errExecutorAtCapacity
-		}
-		lastErr = err
-
-		if attempt < maxAttempts {
-			r.appendActivity(JobActivity{
-				JobID:      job.JobId,
-				JobType:    job.JobType,
-				Source:     "admin_scheduler",
-				Message:    fmt.Sprintf("retrying job attempt %d/%d after error: %v", attempt, maxAttempts, err),
-				Stage:      "retry",
-				OccurredAt: timeToPtr(time.Now().UTC()),
-			})
-			if !waitForShutdownOrTimer(r.shutdownCh, policy.RetryBackoff) {
-				return fmt.Errorf("plugin is shutting down")
-			}
-		}
-	}
-
-	if lastErr == nil {
-		lastErr = fmt.Errorf("execution failed without an explicit error")
-	}
-	return lastErr
-}
-
-func (r *Plugin) shouldSkipDetectionForWaitingJobs(jobType string, policy schedulerPolicy) (bool, int, int) {
-	waitingCount := r.countWaitingTrackedJobs(jobType)
-	threshold := waitingBacklogThreshold(policy)
-	if threshold <= 0 {
-		return false, waitingCount, threshold
-	}
-	return waitingCount >= threshold, waitingCount, threshold
-}
-
-func (r *Plugin) countWaitingTrackedJobs(jobType string) int {
-	normalizedJobType := strings.TrimSpace(jobType)
-	if normalizedJobType == "" {
-		return 0
-	}
-
-	waiting := 0
-	r.jobsMu.RLock()
-	for _, job := range r.jobs {
-		if job == nil {
-			continue
-		}
-		if strings.TrimSpace(job.JobType) != normalizedJobType {
-			continue
-		}
-		if !isWaitingTrackedJobState(job.State) {
-			continue
-		}
-		waiting++
-	}
-	r.jobsMu.RUnlock()
-
-	return waiting
-}
-
-func waitingBacklogThreshold(policy schedulerPolicy) int {
-	concurrency := policy.ExecutionConcurrency
-	if concurrency <= 0 {
-		concurrency = defaultScheduledExecutionConcurrency
-	}
-	threshold := concurrency * defaultWaitingBacklogMultiplier
-	if threshold < defaultWaitingBacklogFloor {
-		threshold = defaultWaitingBacklogFloor
-	}
-	if policy.MaxResults > 0 && threshold > int(policy.MaxResults) {
-		threshold = int(policy.MaxResults)
-	}
-	return threshold
-}
-
-func isExecutorAtCapacityError(err error) bool {
-	if err == nil {
-		return false
-	}
-	if errors.Is(err, errExecutorAtCapacity) {
-		return true
-	}
-	return strings.Contains(strings.ToLower(err.Error()), "executor is at capacity")
-}
-
-func buildScheduledJobSpec(jobType string, proposal *plugin_pb.JobProposal, index int) *plugin_pb.JobSpec {
-	now := timestamppb.Now()
-
-	jobID := fmt.Sprintf("%s-scheduled-%d-%d", jobType, now.AsTime().UnixNano(), index)
-
-	job := &plugin_pb.JobSpec{
-		JobId:       jobID,
-		JobType:     jobType,
-		Priority:    plugin_pb.JobPriority_JOB_PRIORITY_NORMAL,
-		Parameters:  map[string]*plugin_pb.ConfigValue{},
-		Labels:      map[string]string{},
-		CreatedAt:   now,
-		ScheduledAt: now,
-	}
-
-	if proposal == nil {
-		return job
-	}
-
-	if proposal.JobType != "" {
-		job.JobType = proposal.JobType
-	}
-	job.Summary = proposal.Summary
-	job.Detail = proposal.Detail
-	if proposal.Priority != plugin_pb.JobPriority_JOB_PRIORITY_UNSPECIFIED {
-		job.Priority = proposal.Priority
-	}
-	job.DedupeKey = proposal.DedupeKey
-	job.Parameters = CloneConfigValueMap(proposal.Parameters)
-	if proposal.Labels != nil {
-		job.Labels = make(map[string]string, len(proposal.Labels))
-		for k, v := range proposal.Labels {
-			job.Labels[k] = v
-		}
-	}
-	if proposal.NotBefore != nil {
-		job.ScheduledAt = proposal.NotBefore
-	}
-
-	return job
-}
-
-func durationFromSeconds(seconds int32, defaultValue time.Duration) time.Duration {
-	if seconds <= 0 {
-		return defaultValue
-	}
-	return time.Duration(seconds) * time.Second
-}
-
-func secondsFromDuration(duration time.Duration) int32 {
-	if duration <= 0 {
-		return 0
-	}
-	return int32(duration / time.Second)
-}
-
-func waitForShutdownOrTimer(shutdown <-chan struct{}, duration time.Duration) bool {
-	if duration <= 0 {
-		return true
-	}
-
-	timer := time.NewTimer(duration)
-	defer timer.Stop()
-
-	select {
-	case <-shutdown:
-		return false
-	case <-timer.C:
-		return true
-	}
-}
-
-func (r *Plugin) filterProposalsWithActiveJobs(jobType string, proposals []*plugin_pb.JobProposal) ([]*plugin_pb.JobProposal, int) {
-	if len(proposals) == 0 {
-		return proposals, 0
-	}
-
-	activeKeys := make(map[string]struct{})
-	r.jobsMu.RLock()
-	for _, job := range r.jobs {
-		if job == nil {
-			continue
-		}
-		if strings.TrimSpace(job.JobType) != strings.TrimSpace(jobType) {
-			continue
-		}
-		if !isActiveTrackedJobState(job.State) {
-			continue
-		}
-
-		key := strings.TrimSpace(job.DedupeKey)
-		if key == "" {
-			key = strings.TrimSpace(job.JobID)
-		}
-		if key == "" {
-			continue
-		}
-		activeKeys[key] = struct{}{}
-	}
-	r.jobsMu.RUnlock()
-
-	if len(activeKeys) == 0 {
-		return proposals, 0
-	}
-
-	filtered := make([]*plugin_pb.JobProposal, 0, len(proposals))
-	skipped := 0
-	for _, proposal := range proposals {
-		if proposal == nil {
-			continue
-		}
-		key := proposalExecutionKey(proposal)
-		if key != "" {
-			if _, exists := activeKeys[key]; exists {
-				skipped++
-				continue
-			}
-		}
-		filtered = append(filtered, proposal)
-	}
-
-	return filtered, skipped
-}
-
-func proposalExecutionKey(proposal *plugin_pb.JobProposal) string {
-	if proposal == nil {
-		return ""
-	}
-	key := strings.TrimSpace(proposal.DedupeKey)
-	if key != "" {
-		return key
-	}
-	return strings.TrimSpace(proposal.ProposalId)
-}
-
-func isActiveTrackedJobState(state string) bool {
-	normalized := strings.ToLower(strings.TrimSpace(state))
-	switch normalized {
-	case "pending", "assigned", "running", "in_progress", "job_state_pending", "job_state_assigned", "job_state_running":
-		return true
-	default:
-		return false
-	}
-}
-
-func isWaitingTrackedJobState(state string) bool {
-	normalized := strings.ToLower(strings.TrimSpace(state))
-	return normalized == "pending" || normalized == "job_state_pending"
-}
-
-func (r *Plugin) filterScheduledProposals(proposals []*plugin_pb.JobProposal) []*plugin_pb.JobProposal {
-	filtered := make([]*plugin_pb.JobProposal, 0, len(proposals))
-	seenInRun := make(map[string]struct{}, len(proposals))
-
-	for _, proposal := range proposals {
-		if proposal == nil {
-			continue
-		}
-
-		key := proposal.DedupeKey
-		if key == "" {
-			key = proposal.ProposalId
-		}
-		if key == "" {
-			filtered = append(filtered, proposal)
-			continue
-		}
-
-		if _, exists := seenInRun[key]; exists {
-			continue
-		}
-
-		seenInRun[key] = struct{}{}
-		filtered = append(filtered, proposal)
-	}
-
-	return filtered
-}

weed/admin/plugin/plugin_scheduler_test.go

@@ -1,583 +0,0 @@
-package plugin
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-)
-
-func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
-		JobType: "vacuum",
-		AdminRuntime: &plugin_pb.AdminRuntimeConfig{
-			Enabled:                       true,
-			DetectionIntervalSeconds:      30,
-			DetectionTimeoutSeconds:       20,
-			MaxJobsPerDetection:           123,
-			GlobalExecutionConcurrency:    5,
-			PerWorkerExecutionConcurrency: 2,
-			RetryLimit:                    4,
-			RetryBackoffSeconds:           7,
-		},
-	})
-	if err != nil {
-		t.Fatalf("SaveJobTypeConfig: %v", err)
-	}
-
-	policy, enabled, err := pluginSvc.loadSchedulerPolicy("vacuum")
-	if err != nil {
-		t.Fatalf("loadSchedulerPolicy: %v", err)
-	}
-	if !enabled {
-		t.Fatalf("expected enabled policy")
-	}
-	if policy.MaxResults != 123 {
-		t.Fatalf("unexpected max results: got=%d", policy.MaxResults)
-	}
-	if policy.ExecutionConcurrency != 5 {
-		t.Fatalf("unexpected global concurrency: got=%d", policy.ExecutionConcurrency)
-	}
-	if policy.PerWorkerConcurrency != 2 {
-		t.Fatalf("unexpected per-worker concurrency: got=%d", policy.PerWorkerConcurrency)
-	}
-	if policy.RetryLimit != 4 {
-		t.Fatalf("unexpected retry limit: got=%d", policy.RetryLimit)
-	}
-}
-
-func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	err = pluginSvc.store.SaveDescriptor("ec", &plugin_pb.JobTypeDescriptor{
-		JobType: "ec",
-		AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
-			Enabled:                       true,
-			DetectionIntervalSeconds:      60,
-			DetectionTimeoutSeconds:       25,
-			MaxJobsPerDetection:           30,
-			GlobalExecutionConcurrency:    4,
-			PerWorkerExecutionConcurrency: 2,
-			RetryLimit:                    3,
-			RetryBackoffSeconds:           6,
-		},
-	})
-	if err != nil {
-		t.Fatalf("SaveDescriptor: %v", err)
-	}
-
-	policy, enabled, err := pluginSvc.loadSchedulerPolicy("ec")
-	if err != nil {
-		t.Fatalf("loadSchedulerPolicy: %v", err)
-	}
-	if !enabled {
-		t.Fatalf("expected enabled policy from descriptor defaults")
-	}
-	if policy.MaxResults != 30 {
-		t.Fatalf("unexpected max results: got=%d", policy.MaxResults)
-	}
-	if policy.ExecutionConcurrency != 4 {
-		t.Fatalf("unexpected global concurrency: got=%d", policy.ExecutionConcurrency)
-	}
-	if policy.PerWorkerConcurrency != 2 {
-		t.Fatalf("unexpected per-worker concurrency: got=%d", policy.PerWorkerConcurrency)
-	}
-}
-
-func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 4},
-		},
-	})
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-b",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 2},
-		},
-	})
-
-	policy := schedulerPolicy{
-		PerWorkerConcurrency:   1,
-		ExecutorReserveBackoff: time.Millisecond,
-	}
-
-	executor1, release1, err := pluginSvc.reserveScheduledExecutor("balance", policy)
-	if err != nil {
-		t.Fatalf("reserve executor 1: %v", err)
-	}
-	defer release1()
-
-	executor2, release2, err := pluginSvc.reserveScheduledExecutor("balance", policy)
-	if err != nil {
-		t.Fatalf("reserve executor 2: %v", err)
-	}
-	defer release2()
-
-	if executor1.WorkerID == executor2.WorkerID {
-		t.Fatalf("expected different executors due per-worker limit, got same worker %s", executor1.WorkerID)
-	}
-}
-
-func TestFilterScheduledProposalsDedupe(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	proposals := []*plugin_pb.JobProposal{
-		{ProposalId: "p1", DedupeKey: "d1"},
-		{ProposalId: "p2", DedupeKey: "d1"}, // same dedupe key
-		{ProposalId: "p3", DedupeKey: "d3"},
-		{ProposalId: "p3"}, // fallback dedupe by proposal id
-		{ProposalId: "p4"},
-		{ProposalId: "p4"}, // same proposal id, no dedupe key
-	}
-
-	filtered := pluginSvc.filterScheduledProposals(proposals)
-	if len(filtered) != 4 {
-		t.Fatalf("unexpected filtered size: got=%d want=4", len(filtered))
-	}
-
-	filtered2 := pluginSvc.filterScheduledProposals(proposals)
-	if len(filtered2) != 4 {
-		t.Fatalf("expected second run dedupe to be per-run only, got=%d", len(filtered2))
-	}
-}
-
-func TestBuildScheduledJobSpecDoesNotReuseProposalID(t *testing.T) {
-	t.Parallel()
-
-	proposal := &plugin_pb.JobProposal{
-		ProposalId: "vacuum-2",
-		DedupeKey:  "vacuum:2",
-		JobType:    "vacuum",
-	}
-
-	jobA := buildScheduledJobSpec("vacuum", proposal, 0)
-	jobB := buildScheduledJobSpec("vacuum", proposal, 1)
-
-	if jobA.JobId == proposal.ProposalId {
-		t.Fatalf("scheduled job id must not reuse proposal id: %s", jobA.JobId)
-	}
-	if jobB.JobId == proposal.ProposalId {
-		t.Fatalf("scheduled job id must not reuse proposal id: %s", jobB.JobId)
-	}
-	if jobA.JobId == jobB.JobId {
-		t.Fatalf("scheduled job ids must be unique across jobs: %s", jobA.JobId)
-	}
-}
-
-func TestFilterProposalsWithActiveJobs(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	pluginSvc.trackExecutionStart("req-1", "worker-a", &plugin_pb.JobSpec{
-		JobId:     "job-1",
-		JobType:   "vacuum",
-		DedupeKey: "vacuum:k1",
-	}, 1)
-	pluginSvc.trackExecutionStart("req-2", "worker-b", &plugin_pb.JobSpec{
-		JobId:   "job-2",
-		JobType: "vacuum",
-	}, 1)
-	pluginSvc.trackExecutionQueued(&plugin_pb.JobSpec{
-		JobId:     "job-3",
-		JobType:   "vacuum",
-		DedupeKey: "vacuum:k4",
-	})
-
-	filtered, skipped := pluginSvc.filterProposalsWithActiveJobs("vacuum", []*plugin_pb.JobProposal{
-		{ProposalId: "proposal-1", JobType: "vacuum", DedupeKey: "vacuum:k1"},
-		{ProposalId: "job-2", JobType: "vacuum"},
-		{ProposalId: "proposal-2b", JobType: "vacuum", DedupeKey: "vacuum:k4"},
-		{ProposalId: "proposal-3", JobType: "vacuum", DedupeKey: "vacuum:k3"},
-		{ProposalId: "proposal-4", JobType: "balance", DedupeKey: "balance:k1"},
-	})
-	if skipped != 3 {
-		t.Fatalf("unexpected skipped count: got=%d want=3", skipped)
-	}
-	if len(filtered) != 2 {
-		t.Fatalf("unexpected filtered size: got=%d want=2", len(filtered))
-	}
-	if filtered[0].ProposalId != "proposal-3" || filtered[1].ProposalId != "proposal-4" {
-		t.Fatalf("unexpected filtered proposals: got=%s,%s", filtered[0].ProposalId, filtered[1].ProposalId)
-	}
-}
-
-func TestReserveScheduledExecutorTimesOutWhenNoExecutor(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	policy := schedulerPolicy{
-		ExecutionTimeout:       30 * time.Millisecond,
-		ExecutorReserveBackoff: 5 * time.Millisecond,
-		PerWorkerConcurrency:   1,
-	}
-
-	start := time.Now()
-	pluginSvc.Shutdown()
-	_, _, err = pluginSvc.reserveScheduledExecutor("missing-job-type", policy)
-	if err == nil {
-		t.Fatalf("expected reservation shutdown error")
-	}
-	if time.Since(start) > 50*time.Millisecond {
-		t.Fatalf("reservation returned too late after shutdown: duration=%v", time.Since(start))
-	}
-}
-
-func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 1},
-		},
-	})
-
-	policy := schedulerPolicy{
-		ExecutionTimeout:       time.Second,
-		PerWorkerConcurrency:   8,
-		ExecutorReserveBackoff: 5 * time.Millisecond,
-	}
-
-	_, release1, err := pluginSvc.reserveScheduledExecutor("balance", policy)
-	if err != nil {
-		t.Fatalf("reserve executor 1: %v", err)
-	}
-	defer release1()
-
-	type reserveResult struct {
-		err error
-	}
-	secondReserveCh := make(chan reserveResult, 1)
-	go func() {
-		_, release2, reserveErr := pluginSvc.reserveScheduledExecutor("balance", policy)
-		if release2 != nil {
-			release2()
-		}
-		secondReserveCh <- reserveResult{err: reserveErr}
-	}()
-
-	select {
-	case result := <-secondReserveCh:
-		t.Fatalf("expected second reservation to wait for capacity, got=%v", result.err)
-	case <-time.After(25 * time.Millisecond):
-		// Expected: still waiting.
-	}
-
-	release1()
-
-	select {
-	case result := <-secondReserveCh:
-		if result.err != nil {
-			t.Fatalf("second reservation error: %v", result.err)
-		}
-	case <-time.After(200 * time.Millisecond):
-		t.Fatalf("second reservation did not acquire after capacity release")
-	}
-}
-
-func TestShouldSkipDetectionForWaitingJobs(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	policy := schedulerPolicy{
-		ExecutionConcurrency: 2,
-		MaxResults:           100,
-	}
-	threshold := waitingBacklogThreshold(policy)
-	if threshold <= 0 {
-		t.Fatalf("expected positive waiting threshold")
-	}
-
-	for i := 0; i < threshold; i++ {
-		pluginSvc.trackExecutionQueued(&plugin_pb.JobSpec{
-			JobId:     fmt.Sprintf("job-waiting-%d", i),
-			JobType:   "vacuum",
-			DedupeKey: fmt.Sprintf("vacuum:%d", i),
-		})
-	}
-
-	skip, waitingCount, waitingThreshold := pluginSvc.shouldSkipDetectionForWaitingJobs("vacuum", policy)
-	if !skip {
-		t.Fatalf("expected detection to skip when waiting backlog reaches threshold")
-	}
-	if waitingCount != threshold {
-		t.Fatalf("unexpected waiting count: got=%d want=%d", waitingCount, threshold)
-	}
-	if waitingThreshold != threshold {
-		t.Fatalf("unexpected waiting threshold: got=%d want=%d", waitingThreshold, threshold)
-	}
-}
-
-func TestWaitingBacklogThresholdHonorsMaxResultsCap(t *testing.T) {
-	t.Parallel()
-
-	policy := schedulerPolicy{
-		ExecutionConcurrency: 8,
-		MaxResults:           6,
-	}
-	threshold := waitingBacklogThreshold(policy)
-	if threshold != 6 {
-		t.Fatalf("expected threshold to be capped by max results, got=%d", threshold)
-	}
-}
-
-func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	const jobType = "vacuum"
-	err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
-		JobType: jobType,
-		AdminRuntime: &plugin_pb.AdminRuntimeConfig{
-			Enabled:                       true,
-			DetectionIntervalSeconds:      45,
-			DetectionTimeoutSeconds:       30,
-			MaxJobsPerDetection:           80,
-			GlobalExecutionConcurrency:    3,
-			PerWorkerExecutionConcurrency: 2,
-			RetryLimit:                    1,
-			RetryBackoffSeconds:           9,
-		},
-	})
-	if err != nil {
-		t.Fatalf("SaveJobTypeConfig: %v", err)
-	}
-
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: jobType, CanDetect: true, CanExecute: true},
-		},
-	})
-
-	nextDetectionAt := time.Now().UTC().Add(2 * time.Minute).Round(time.Second)
-	pluginSvc.schedulerMu.Lock()
-	pluginSvc.nextDetectionAt[jobType] = nextDetectionAt
-	pluginSvc.detectionInFlight[jobType] = true
-	pluginSvc.schedulerMu.Unlock()
-
-	states, err := pluginSvc.ListSchedulerStates()
-	if err != nil {
-		t.Fatalf("ListSchedulerStates: %v", err)
-	}
-
-	state := findSchedulerState(states, jobType)
-	if state == nil {
-		t.Fatalf("missing scheduler state for %s", jobType)
-	}
-	if !state.Enabled {
-		t.Fatalf("expected enabled scheduler state")
-	}
-	if state.PolicyError != "" {
-		t.Fatalf("unexpected policy error: %s", state.PolicyError)
-	}
-	if !state.DetectionInFlight {
-		t.Fatalf("expected detection in flight")
-	}
-	if state.NextDetectionAt == nil {
-		t.Fatalf("expected next detection time")
-	}
-	if state.NextDetectionAt.Unix() != nextDetectionAt.Unix() {
-		t.Fatalf("unexpected next detection time: got=%v want=%v", state.NextDetectionAt, nextDetectionAt)
-	}
-	if state.DetectionIntervalSeconds != 45 {
-		t.Fatalf("unexpected detection interval: got=%d", state.DetectionIntervalSeconds)
-	}
-	if state.DetectionTimeoutSeconds != 30 {
-		t.Fatalf("unexpected detection timeout: got=%d", state.DetectionTimeoutSeconds)
-	}
-	if state.ExecutionTimeoutSeconds != 90 {
-		t.Fatalf("unexpected execution timeout: got=%d", state.ExecutionTimeoutSeconds)
-	}
-	if state.MaxJobsPerDetection != 80 {
-		t.Fatalf("unexpected max jobs per detection: got=%d", state.MaxJobsPerDetection)
-	}
-	if state.GlobalExecutionConcurrency != 3 {
-		t.Fatalf("unexpected global execution concurrency: got=%d", state.GlobalExecutionConcurrency)
-	}
-	if state.PerWorkerExecutionConcurrency != 2 {
-		t.Fatalf("unexpected per worker execution concurrency: got=%d", state.PerWorkerExecutionConcurrency)
-	}
-	if state.RetryLimit != 1 {
-		t.Fatalf("unexpected retry limit: got=%d", state.RetryLimit)
-	}
-	if state.RetryBackoffSeconds != 9 {
-		t.Fatalf("unexpected retry backoff: got=%d", state.RetryBackoffSeconds)
-	}
-	if !state.DetectorAvailable || state.DetectorWorkerID != "worker-a" {
-		t.Fatalf("unexpected detector assignment: available=%v worker=%s", state.DetectorAvailable, state.DetectorWorkerID)
-	}
-	if state.ExecutorWorkerCount != 1 {
-		t.Fatalf("unexpected executor worker count: got=%d", state.ExecutorWorkerCount)
-	}
-}
-
-func TestListSchedulerStatesShowsDisabledWhenNoPolicy(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	const jobType = "balance"
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-b",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: jobType, CanDetect: true, CanExecute: true},
-		},
-	})
-
-	states, err := pluginSvc.ListSchedulerStates()
-	if err != nil {
-		t.Fatalf("ListSchedulerStates: %v", err)
-	}
-
-	state := findSchedulerState(states, jobType)
-	if state == nil {
-		t.Fatalf("missing scheduler state for %s", jobType)
-	}
-	if state.Enabled {
-		t.Fatalf("expected disabled scheduler state")
-	}
-	if state.PolicyError != "" {
-		t.Fatalf("unexpected policy error: %s", state.PolicyError)
-	}
-	if !state.DetectorAvailable || state.DetectorWorkerID != "worker-b" {
-		t.Fatalf("unexpected detector details: available=%v worker=%s", state.DetectorAvailable, state.DetectorWorkerID)
-	}
-	if state.ExecutorWorkerCount != 1 {
-		t.Fatalf("unexpected executor worker count: got=%d", state.ExecutorWorkerCount)
-	}
-}
-
-func findSchedulerState(states []SchedulerJobTypeState, jobType string) *SchedulerJobTypeState {
-	for i := range states {
-		if states[i].JobType == jobType {
-			return &states[i]
-		}
-	}
-	return nil
-}
-
-func TestPickDetectorPrefersLeasedWorker(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "vacuum", CanDetect: true},
-		},
-	})
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-b",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "vacuum", CanDetect: true},
-		},
-	})
-
-	pluginSvc.setDetectorLease("vacuum", "worker-b")
-
-	detector, err := pluginSvc.pickDetector("vacuum")
-	if err != nil {
-		t.Fatalf("pickDetector: %v", err)
-	}
-	if detector.WorkerID != "worker-b" {
-		t.Fatalf("expected leased detector worker-b, got=%s", detector.WorkerID)
-	}
-}
-
-func TestPickDetectorReassignsWhenLeaseIsStale(t *testing.T) {
-	t.Parallel()
-
-	pluginSvc, err := New(Options{})
-	if err != nil {
-		t.Fatalf("New: %v", err)
-	}
-	defer pluginSvc.Shutdown()
-
-	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "vacuum", CanDetect: true},
-		},
-	})
-	pluginSvc.setDetectorLease("vacuum", "worker-stale")
-
-	detector, err := pluginSvc.pickDetector("vacuum")
-	if err != nil {
-		t.Fatalf("pickDetector: %v", err)
-	}
-	if detector.WorkerID != "worker-a" {
-		t.Fatalf("expected reassigned detector worker-a, got=%s", detector.WorkerID)
-	}
-
-	lease := pluginSvc.getDetectorLease("vacuum")
-	if lease != "worker-a" {
-		t.Fatalf("expected detector lease to be updated to worker-a, got=%s", lease)
-	}
-}

weed/admin/plugin/plugin_schema_prefetch.go

@@ -1,66 +0,0 @@
-package plugin
-
-import (
-	"context"
-	"sort"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-)
-
-const descriptorPrefetchTimeout = 20 * time.Second
-
-func (r *Plugin) prefetchDescriptorsFromHello(hello *plugin_pb.WorkerHello) {
-	if hello == nil || len(hello.Capabilities) == 0 {
-		return
-	}
-
-	jobTypeSet := make(map[string]struct{})
-	for _, capability := range hello.Capabilities {
-		if capability == nil || capability.JobType == "" {
-			continue
-		}
-		if !capability.CanDetect && !capability.CanExecute {
-			continue
-		}
-		jobTypeSet[capability.JobType] = struct{}{}
-	}
-
-	if len(jobTypeSet) == 0 {
-		return
-	}
-
-	jobTypes := make([]string, 0, len(jobTypeSet))
-	for jobType := range jobTypeSet {
-		jobTypes = append(jobTypes, jobType)
-	}
-	sort.Strings(jobTypes)
-
-	for _, jobType := range jobTypes {
-		select {
-		case <-r.shutdownCh:
-			return
-		default:
-		}
-
-		descriptor, err := r.store.LoadDescriptor(jobType)
-		if err != nil {
-			glog.Warningf("Plugin descriptor prefetch check failed for %s: %v", jobType, err)
-			continue
-		}
-		if descriptor != nil {
-			continue
-		}
-
-		ctx, cancel := context.WithTimeout(r.ctx, descriptorPrefetchTimeout)
-		_, err = r.RequestConfigSchema(ctx, jobType, false)
-		cancel()
-		if err != nil {
-			glog.V(1).Infof("Plugin descriptor prefetch skipped for %s: %v", jobType, err)
-			continue
-		}
-
-		glog.V(1).Infof("Plugin descriptor prefetched for job_type=%s", jobType)
-	}
-}

weed/admin/plugin/registry.go

@@ -1,465 +0,0 @@
-package plugin
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-)
-
-const defaultWorkerStaleTimeout = 2 * time.Minute
-
-// WorkerSession contains tracked worker metadata and plugin status.
-type WorkerSession struct {
-	WorkerID        string
-	WorkerInstance  string
-	Address         string
-	WorkerVersion   string
-	ProtocolVersion string
-	ConnectedAt     time.Time
-	LastSeenAt      time.Time
-	Capabilities    map[string]*plugin_pb.JobTypeCapability
-	Heartbeat       *plugin_pb.WorkerHeartbeat
-}
-
-// Registry tracks connected plugin workers and capability-based selection.
-type Registry struct {
-	mu             sync.RWMutex
-	sessions       map[string]*WorkerSession
-	staleAfter     time.Duration
-	detectorCursor map[string]int
-	executorCursor map[string]int
-}
-
-func NewRegistry() *Registry {
-	return &Registry{
-		sessions:       make(map[string]*WorkerSession),
-		staleAfter:     defaultWorkerStaleTimeout,
-		detectorCursor: make(map[string]int),
-		executorCursor: make(map[string]int),
-	}
-}
-
-func (r *Registry) UpsertFromHello(hello *plugin_pb.WorkerHello) *WorkerSession {
-	now := time.Now()
-	caps := make(map[string]*plugin_pb.JobTypeCapability, len(hello.Capabilities))
-	for _, c := range hello.Capabilities {
-		if c == nil || c.JobType == "" {
-			continue
-		}
-		caps[c.JobType] = cloneJobTypeCapability(c)
-	}
-
-	r.mu.Lock()
-	defer r.mu.Unlock()
-
-	session, ok := r.sessions[hello.WorkerId]
-	if !ok {
-		session = &WorkerSession{
-			WorkerID:    hello.WorkerId,
-			ConnectedAt: now,
-		}
-		r.sessions[hello.WorkerId] = session
-	}
-
-	session.WorkerInstance = hello.WorkerInstanceId
-	session.Address = hello.Address
-	session.WorkerVersion = hello.WorkerVersion
-	session.ProtocolVersion = hello.ProtocolVersion
-	session.LastSeenAt = now
-	session.Capabilities = caps
-
-	return cloneWorkerSession(session)
-}
-
-func (r *Registry) Remove(workerID string) {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	delete(r.sessions, workerID)
-}
-
-func (r *Registry) UpdateHeartbeat(workerID string, heartbeat *plugin_pb.WorkerHeartbeat) {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-
-	session, ok := r.sessions[workerID]
-	if !ok {
-		return
-	}
-	session.Heartbeat = cloneWorkerHeartbeat(heartbeat)
-	session.LastSeenAt = time.Now()
-}
-
-func (r *Registry) Get(workerID string) (*WorkerSession, bool) {
-	r.mu.RLock()
-	defer r.mu.RUnlock()
-	session, ok := r.sessions[workerID]
-	if !ok || r.isSessionStaleLocked(session, time.Now()) {
-		return nil, false
-	}
-	return cloneWorkerSession(session), true
-}
-
-func (r *Registry) List() []*WorkerSession {
-	r.mu.RLock()
-	defer r.mu.RUnlock()
-	out := make([]*WorkerSession, 0, len(r.sessions))
-	now := time.Now()
-	for _, s := range r.sessions {
-		if r.isSessionStaleLocked(s, now) {
-			continue
-		}
-		out = append(out, cloneWorkerSession(s))
-	}
-	sort.Slice(out, func(i, j int) bool {
-		return out[i].WorkerID < out[j].WorkerID
-	})
-	return out
-}
-
-// DetectableJobTypes returns sorted job types that currently have at least one detect-capable worker.
-func (r *Registry) DetectableJobTypes() []string {
-	r.mu.RLock()
-	defer r.mu.RUnlock()
-
-	jobTypes := make(map[string]struct{})
-	now := time.Now()
-	for _, session := range r.sessions {
-		if r.isSessionStaleLocked(session, now) {
-			continue
-		}
-		for jobType, capability := range session.Capabilities {
-			if capability == nil || !capability.CanDetect {
-				continue
-			}
-			jobTypes[jobType] = struct{}{}
-		}
-	}
-
-	out := make([]string, 0, len(jobTypes))
-	for jobType := range jobTypes {
-		out = append(out, jobType)
-	}
-	sort.Strings(out)
-	return out
-}
-
-// JobTypes returns sorted job types known by connected workers regardless of capability kind.
-func (r *Registry) JobTypes() []string {
-	r.mu.RLock()
-	defer r.mu.RUnlock()
-
-	jobTypes := make(map[string]struct{})
-	now := time.Now()
-	for _, session := range r.sessions {
-		if r.isSessionStaleLocked(session, now) {
-			continue
-		}
-		for jobType := range session.Capabilities {
-			if jobType == "" {
-				continue
-			}
-			jobTypes[jobType] = struct{}{}
-		}
-	}
-
-	out := make([]string, 0, len(jobTypes))
-	for jobType := range jobTypes {
-		out = append(out, jobType)
-	}
-	sort.Strings(out)
-	return out
-}
-
-// PickSchemaProvider picks one worker for schema requests.
-// Preference order:
-// 1) workers that can detect this job type
-// 2) workers that can execute this job type
-// tie-break: more free slots, then lexical worker ID.
-func (r *Registry) PickSchemaProvider(jobType string) (*WorkerSession, error) {
-	r.mu.RLock()
-	defer r.mu.RUnlock()
-
-	var candidates []*WorkerSession
-	now := time.Now()
-	for _, s := range r.sessions {
-		if r.isSessionStaleLocked(s, now) {
-			continue
-		}
-		capability := s.Capabilities[jobType]
-		if capability == nil {
-			continue
-		}
-		if capability.CanDetect || capability.CanExecute {
-			candidates = append(candidates, s)
-		}
-	}
-
-	if len(candidates) == 0 {
-		return nil, fmt.Errorf("no worker available for schema job_type=%s", jobType)
-	}
-
-	sort.Slice(candidates, func(i, j int) bool {
-		a := candidates[i]
-		b := candidates[j]
-		ac := a.Capabilities[jobType]
-		bc := b.Capabilities[jobType]
-
-		// Prefer detect-capable providers first.
-		if ac.CanDetect != bc.CanDetect {
-			return ac.CanDetect
-		}
-
-		aSlots := availableDetectionSlots(a, ac) + availableExecutionSlots(a, ac)
-		bSlots := availableDetectionSlots(b, bc) + availableExecutionSlots(b, bc)
-		if aSlots != bSlots {
-			return aSlots > bSlots
-		}
-		return a.WorkerID < b.WorkerID
-	})
-
-	return cloneWorkerSession(candidates[0]), nil
-}
-
-// PickDetector picks one detector worker for a job type.
-func (r *Registry) PickDetector(jobType string) (*WorkerSession, error) {
-	return r.pickByKind(jobType, true)
-}
-
-// PickExecutor picks one executor worker for a job type.
-func (r *Registry) PickExecutor(jobType string) (*WorkerSession, error) {
-	return r.pickByKind(jobType, false)
-}
-
-// ListExecutors returns sorted executor candidates for one job type.
-// Ordering is by most available execution slots, then lexical worker ID.
-// The top tie group is rotated round-robin to prevent sticky assignment.
-func (r *Registry) ListExecutors(jobType string) ([]*WorkerSession, error) {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-
-	candidates := r.collectByKindLocked(jobType, false, time.Now())
-	if len(candidates) == 0 {
-		return nil, fmt.Errorf("no executor worker available for job_type=%s", jobType)
-	}
-
-	sortByKind(candidates, jobType, false)
-	r.rotateTopCandidatesLocked(candidates, jobType, false)
-
-	out := make([]*WorkerSession, 0, len(candidates))
-	for _, candidate := range candidates {
-		out = append(out, cloneWorkerSession(candidate))
-	}
-	return out, nil
-}
-
-func (r *Registry) pickByKind(jobType string, detect bool) (*WorkerSession, error) {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-
-	candidates := r.collectByKindLocked(jobType, detect, time.Now())
-
-	if len(candidates) == 0 {
-		kind := "executor"
-		if detect {
-			kind = "detector"
-		}
-		return nil, fmt.Errorf("no %s worker available for job_type=%s", kind, jobType)
-	}
-
-	sortByKind(candidates, jobType, detect)
-	r.rotateTopCandidatesLocked(candidates, jobType, detect)
-
-	return cloneWorkerSession(candidates[0]), nil
-}
-
-func (r *Registry) collectByKindLocked(jobType string, detect bool, now time.Time) []*WorkerSession {
-	var candidates []*WorkerSession
-	for _, session := range r.sessions {
-		if r.isSessionStaleLocked(session, now) {
-			continue
-		}
-		capability := session.Capabilities[jobType]
-		if capability == nil {
-			continue
-		}
-		if detect && capability.CanDetect {
-			candidates = append(candidates, session)
-		}
-		if !detect && capability.CanExecute {
-			candidates = append(candidates, session)
-		}
-	}
-	return candidates
-}
-
-func (r *Registry) isSessionStaleLocked(session *WorkerSession, now time.Time) bool {
-	if session == nil {
-		return true
-	}
-	if r.staleAfter <= 0 {
-		return false
-	}
-
-	lastSeen := session.LastSeenAt
-	if lastSeen.IsZero() {
-		lastSeen = session.ConnectedAt
-	}
-	if lastSeen.IsZero() {
-		return false
-	}
-	return now.Sub(lastSeen) > r.staleAfter
-}
-
-func sortByKind(candidates []*WorkerSession, jobType string, detect bool) {
-	sort.Slice(candidates, func(i, j int) bool {
-		a := candidates[i]
-		b := candidates[j]
-		ac := a.Capabilities[jobType]
-		bc := b.Capabilities[jobType]
-
-		aSlots := availableSlotsByKind(a, ac, detect)
-		bSlots := availableSlotsByKind(b, bc, detect)
-
-		if aSlots != bSlots {
-			return aSlots > bSlots
-		}
-		return a.WorkerID < b.WorkerID
-	})
-}
-
-func (r *Registry) rotateTopCandidatesLocked(candidates []*WorkerSession, jobType string, detect bool) {
-	if len(candidates) < 2 {
-		return
-	}
-
-	capability := candidates[0].Capabilities[jobType]
-	topSlots := availableSlotsByKind(candidates[0], capability, detect)
-	tieEnd := 1
-	for tieEnd < len(candidates) {
-		nextCapability := candidates[tieEnd].Capabilities[jobType]
-		if availableSlotsByKind(candidates[tieEnd], nextCapability, detect) != topSlots {
-			break
-		}
-		tieEnd++
-	}
-	if tieEnd <= 1 {
-		return
-	}
-
-	cursorKey := strings.TrimSpace(jobType)
-	if cursorKey == "" {
-		cursorKey = "*"
-	}
-
-	var offset int
-	if detect {
-		offset = r.detectorCursor[cursorKey] % tieEnd
-		r.detectorCursor[cursorKey] = (offset + 1) % tieEnd
-	} else {
-		offset = r.executorCursor[cursorKey] % tieEnd
-		r.executorCursor[cursorKey] = (offset + 1) % tieEnd
-	}
-
-	if offset == 0 {
-		return
-	}
-
-	prefix := append([]*WorkerSession(nil), candidates[:tieEnd]...)
-	for i := 0; i < tieEnd; i++ {
-		candidates[i] = prefix[(i+offset)%tieEnd]
-	}
-}
-
-func availableSlotsByKind(
-	session *WorkerSession,
-	capability *plugin_pb.JobTypeCapability,
-	detect bool,
-) int {
-	if detect {
-		return availableDetectionSlots(session, capability)
-	}
-	return availableExecutionSlots(session, capability)
-}
-
-func availableDetectionSlots(session *WorkerSession, capability *plugin_pb.JobTypeCapability) int {
-	if session.Heartbeat != nil && session.Heartbeat.DetectionSlotsTotal > 0 {
-		free := int(session.Heartbeat.DetectionSlotsTotal - session.Heartbeat.DetectionSlotsUsed)
-		if free < 0 {
-			return 0
-		}
-		return free
-	}
-	if capability.MaxDetectionConcurrency > 0 {
-		return int(capability.MaxDetectionConcurrency)
-	}
-	return 1
-}
-
-func availableExecutionSlots(session *WorkerSession, capability *plugin_pb.JobTypeCapability) int {
-	if session.Heartbeat != nil && session.Heartbeat.ExecutionSlotsTotal > 0 {
-		free := int(session.Heartbeat.ExecutionSlotsTotal - session.Heartbeat.ExecutionSlotsUsed)
-		if free < 0 {
-			return 0
-		}
-		return free
-	}
-	if capability.MaxExecutionConcurrency > 0 {
-		return int(capability.MaxExecutionConcurrency)
-	}
-	return 1
-}
-
-func cloneWorkerSession(in *WorkerSession) *WorkerSession {
-	if in == nil {
-		return nil
-	}
-	out := *in
-	out.Capabilities = make(map[string]*plugin_pb.JobTypeCapability, len(in.Capabilities))
-	for jobType, cap := range in.Capabilities {
-		out.Capabilities[jobType] = cloneJobTypeCapability(cap)
-	}
-	out.Heartbeat = cloneWorkerHeartbeat(in.Heartbeat)
-	return &out
-}
-
-func cloneJobTypeCapability(in *plugin_pb.JobTypeCapability) *plugin_pb.JobTypeCapability {
-	if in == nil {
-		return nil
-	}
-	out := *in
-	return &out
-}
-
-func cloneWorkerHeartbeat(in *plugin_pb.WorkerHeartbeat) *plugin_pb.WorkerHeartbeat {
-	if in == nil {
-		return nil
-	}
-	out := *in
-	if in.RunningWork != nil {
-		out.RunningWork = make([]*plugin_pb.RunningWork, 0, len(in.RunningWork))
-		for _, rw := range in.RunningWork {
-			if rw == nil {
-				continue
-			}
-			clone := *rw
-			out.RunningWork = append(out.RunningWork, &clone)
-		}
-	}
-	if in.QueuedJobsByType != nil {
-		out.QueuedJobsByType = make(map[string]int32, len(in.QueuedJobsByType))
-		for k, v := range in.QueuedJobsByType {
-			out.QueuedJobsByType[k] = v
-		}
-	}
-	if in.Metadata != nil {
-		out.Metadata = make(map[string]string, len(in.Metadata))
-		for k, v := range in.Metadata {
-			out.Metadata[k] = v
-		}
-	}
-	return &out
-}

weed/admin/plugin/registry_test.go

@@ -1,321 +0,0 @@
-package plugin
-
-import (
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
-)
-
-func TestRegistryPickDetectorPrefersMoreFreeSlots(t *testing.T) {
-	t.Parallel()
-
-	r := NewRegistry()
-
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "vacuum", CanDetect: true, CanExecute: true, MaxDetectionConcurrency: 2, MaxExecutionConcurrency: 2},
-		},
-	})
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-b",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "vacuum", CanDetect: true, CanExecute: true, MaxDetectionConcurrency: 4, MaxExecutionConcurrency: 4},
-		},
-	})
-
-	r.UpdateHeartbeat("worker-a", &plugin_pb.WorkerHeartbeat{
-		WorkerId:            "worker-a",
-		DetectionSlotsUsed:  1,
-		DetectionSlotsTotal: 2,
-	})
-	r.UpdateHeartbeat("worker-b", &plugin_pb.WorkerHeartbeat{
-		WorkerId:            "worker-b",
-		DetectionSlotsUsed:  1,
-		DetectionSlotsTotal: 4,
-	})
-
-	picked, err := r.PickDetector("vacuum")
-	if err != nil {
-		t.Fatalf("PickDetector: %v", err)
-	}
-	if picked.WorkerID != "worker-b" {
-		t.Fatalf("unexpected detector picked: got %s want worker-b", picked.WorkerID)
-	}
-}
-
-func TestRegistryPickExecutorAllowsSameWorker(t *testing.T) {
-	t.Parallel()
-
-	r := NewRegistry()
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-x",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "balance", CanDetect: true, CanExecute: true, MaxDetectionConcurrency: 1, MaxExecutionConcurrency: 1},
-		},
-	})
-
-	detector, err := r.PickDetector("balance")
-	if err != nil {
-		t.Fatalf("PickDetector: %v", err)
-	}
-	executor, err := r.PickExecutor("balance")
-	if err != nil {
-		t.Fatalf("PickExecutor: %v", err)
-	}
-
-	if detector.WorkerID != "worker-x" || executor.WorkerID != "worker-x" {
-		t.Fatalf("expected same worker for detect/execute, got detector=%s executor=%s", detector.WorkerID, executor.WorkerID)
-	}
-}
-
-func TestRegistryDetectableJobTypes(t *testing.T) {
-	t.Parallel()
-
-	r := NewRegistry()
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "vacuum", CanDetect: true, CanExecute: true},
-			{JobType: "balance", CanDetect: false, CanExecute: true},
-		},
-	})
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-b",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "ec", CanDetect: true, CanExecute: false},
-			{JobType: "vacuum", CanDetect: true, CanExecute: false},
-		},
-	})
-
-	got := r.DetectableJobTypes()
-	want := []string{"ec", "vacuum"}
-	if !reflect.DeepEqual(got, want) {
-		t.Fatalf("unexpected detectable job types: got=%v want=%v", got, want)
-	}
-}
-
-func TestRegistryJobTypes(t *testing.T) {
-	t.Parallel()
-
-	r := NewRegistry()
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "vacuum", CanDetect: true},
-			{JobType: "balance", CanExecute: true},
-		},
-	})
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-b",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "ec", CanDetect: true},
-		},
-	})
-
-	got := r.JobTypes()
-	want := []string{"balance", "ec", "vacuum"}
-	if !reflect.DeepEqual(got, want) {
-		t.Fatalf("unexpected job types: got=%v want=%v", got, want)
-	}
-}
-
-func TestRegistryListExecutorsSortedBySlots(t *testing.T) {
-	t.Parallel()
-
-	r := NewRegistry()
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 2},
-		},
-	})
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-b",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 4},
-		},
-	})
-
-	r.UpdateHeartbeat("worker-a", &plugin_pb.WorkerHeartbeat{
-		WorkerId:            "worker-a",
-		ExecutionSlotsUsed:  1,
-		ExecutionSlotsTotal: 2,
-	})
-	r.UpdateHeartbeat("worker-b", &plugin_pb.WorkerHeartbeat{
-		WorkerId:            "worker-b",
-		ExecutionSlotsUsed:  1,
-		ExecutionSlotsTotal: 4,
-	})
-
-	executors, err := r.ListExecutors("balance")
-	if err != nil {
-		t.Fatalf("ListExecutors: %v", err)
-	}
-	if len(executors) != 2 {
-		t.Fatalf("unexpected candidate count: got=%d", len(executors))
-	}
-	if executors[0].WorkerID != "worker-b" || executors[1].WorkerID != "worker-a" {
-		t.Fatalf("unexpected executor order: got=%s,%s", executors[0].WorkerID, executors[1].WorkerID)
-	}
-}
-
-func TestRegistryPickExecutorRoundRobinForTopTie(t *testing.T) {
-	t.Parallel()
-
-	r := NewRegistry()
-	for _, workerID := range []string{"worker-a", "worker-b", "worker-c"} {
-		r.UpsertFromHello(&plugin_pb.WorkerHello{
-			WorkerId: workerID,
-			Capabilities: []*plugin_pb.JobTypeCapability{
-				{JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 1},
-			},
-		})
-	}
-
-	got := make([]string, 0, 6)
-	for i := 0; i < 6; i++ {
-		executor, err := r.PickExecutor("balance")
-		if err != nil {
-			t.Fatalf("PickExecutor: %v", err)
-		}
-		got = append(got, executor.WorkerID)
-	}
-
-	want := []string{"worker-a", "worker-b", "worker-c", "worker-a", "worker-b", "worker-c"}
-	if !reflect.DeepEqual(got, want) {
-		t.Fatalf("unexpected pick order: got=%v want=%v", got, want)
-	}
-}
-
-func TestRegistryListExecutorsRoundRobinForTopTie(t *testing.T) {
-	t.Parallel()
-
-	r := NewRegistry()
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 2},
-		},
-	})
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-b",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 2},
-		},
-	})
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-c",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 1},
-		},
-	})
-
-	r.UpdateHeartbeat("worker-a", &plugin_pb.WorkerHeartbeat{
-		WorkerId:            "worker-a",
-		ExecutionSlotsUsed:  0,
-		ExecutionSlotsTotal: 2,
-	})
-	r.UpdateHeartbeat("worker-b", &plugin_pb.WorkerHeartbeat{
-		WorkerId:            "worker-b",
-		ExecutionSlotsUsed:  0,
-		ExecutionSlotsTotal: 2,
-	})
-	r.UpdateHeartbeat("worker-c", &plugin_pb.WorkerHeartbeat{
-		WorkerId:            "worker-c",
-		ExecutionSlotsUsed:  0,
-		ExecutionSlotsTotal: 1,
-	})
-
-	firstCall, err := r.ListExecutors("balance")
-	if err != nil {
-		t.Fatalf("ListExecutors first call: %v", err)
-	}
-	secondCall, err := r.ListExecutors("balance")
-	if err != nil {
-		t.Fatalf("ListExecutors second call: %v", err)
-	}
-	thirdCall, err := r.ListExecutors("balance")
-	if err != nil {
-		t.Fatalf("ListExecutors third call: %v", err)
-	}
-
-	if firstCall[0].WorkerID != "worker-a" || firstCall[1].WorkerID != "worker-b" || firstCall[2].WorkerID != "worker-c" {
-		t.Fatalf("unexpected first executor order: got=%s,%s,%s", firstCall[0].WorkerID, firstCall[1].WorkerID, firstCall[2].WorkerID)
-	}
-	if secondCall[0].WorkerID != "worker-b" || secondCall[1].WorkerID != "worker-a" || secondCall[2].WorkerID != "worker-c" {
-		t.Fatalf("unexpected second executor order: got=%s,%s,%s", secondCall[0].WorkerID, secondCall[1].WorkerID, secondCall[2].WorkerID)
-	}
-	if thirdCall[0].WorkerID != "worker-a" || thirdCall[1].WorkerID != "worker-b" || thirdCall[2].WorkerID != "worker-c" {
-		t.Fatalf("unexpected third executor order: got=%s,%s,%s", thirdCall[0].WorkerID, thirdCall[1].WorkerID, thirdCall[2].WorkerID)
-	}
-}
-
-func TestRegistrySkipsStaleWorkersForSelectionAndListing(t *testing.T) {
-	t.Parallel()
-
-	r := NewRegistry()
-	r.staleAfter = 2 * time.Second
-
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-stale",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "vacuum", CanDetect: true, CanExecute: true},
-		},
-	})
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-fresh",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "vacuum", CanDetect: true, CanExecute: true},
-		},
-	})
-
-	r.mu.Lock()
-	r.sessions["worker-stale"].LastSeenAt = time.Now().Add(-10 * time.Second)
-	r.sessions["worker-fresh"].LastSeenAt = time.Now()
-	r.mu.Unlock()
-
-	picked, err := r.PickDetector("vacuum")
-	if err != nil {
-		t.Fatalf("PickDetector: %v", err)
-	}
-	if picked.WorkerID != "worker-fresh" {
-		t.Fatalf("unexpected detector: got=%s want=worker-fresh", picked.WorkerID)
-	}
-
-	if _, ok := r.Get("worker-stale"); ok {
-		t.Fatalf("expected stale worker to be hidden from Get")
-	}
-	if _, ok := r.Get("worker-fresh"); !ok {
-		t.Fatalf("expected fresh worker from Get")
-	}
-
-	listed := r.List()
-	if len(listed) != 1 || listed[0].WorkerID != "worker-fresh" {
-		t.Fatalf("unexpected listed workers: %+v", listed)
-	}
-}
-
-func TestRegistryReturnsNoDetectorWhenAllWorkersStale(t *testing.T) {
-	t.Parallel()
-
-	r := NewRegistry()
-	r.staleAfter = 2 * time.Second
-
-	r.UpsertFromHello(&plugin_pb.WorkerHello{
-		WorkerId: "worker-a",
-		Capabilities: []*plugin_pb.JobTypeCapability{
-			{JobType: "vacuum", CanDetect: true},
-		},
-	})
-
-	r.mu.Lock()
-	r.sessions["worker-a"].LastSeenAt = time.Now().Add(-10 * time.Second)
-	r.mu.Unlock()
-
-	if _, err := r.PickDetector("vacuum"); err == nil {
-		t.Fatalf("expected no detector when all workers are stale")
-	}
-}

weed/admin/plugin/types.go

@@ -1,103 +0,0 @@
-package plugin
-
-import "time"
-
-const (
-	// Keep exactly the last 10 successful and last 10 error runs per job type.
-	MaxSuccessfulRunHistory = 10
-	MaxErrorRunHistory      = 10
-)
-
-type RunOutcome string
-
-const (
-	RunOutcomeSuccess RunOutcome = "success"
-	RunOutcomeError   RunOutcome = "error"
-)
-
-type JobRunRecord struct {
-	RunID       string     `json:"run_id"`
-	JobID       string     `json:"job_id"`
-	JobType     string     `json:"job_type"`
-	WorkerID    string     `json:"worker_id"`
-	Outcome     RunOutcome `json:"outcome"`
-	Message     string     `json:"message,omitempty"`
-	DurationMs  int64      `json:"duration_ms,omitempty"`
-	CompletedAt *time.Time `json:"completed_at,omitempty"`
-}
-
-type JobTypeRunHistory struct {
-	JobType         string         `json:"job_type"`
-	SuccessfulRuns  []JobRunRecord `json:"successful_runs"`
-	ErrorRuns       []JobRunRecord `json:"error_runs"`
-	LastUpdatedTime *time.Time     `json:"last_updated_time,omitempty"`
-}
-
-type TrackedJob struct {
-	JobID              string                 `json:"job_id"`
-	JobType            string                 `json:"job_type"`
-	RequestID          string                 `json:"request_id"`
-	WorkerID           string                 `json:"worker_id"`
-	DedupeKey          string                 `json:"dedupe_key,omitempty"`
-	Summary            string                 `json:"summary,omitempty"`
-	Detail             string                 `json:"detail,omitempty"`
-	Parameters         map[string]interface{} `json:"parameters,omitempty"`
-	Labels             map[string]string      `json:"labels,omitempty"`
-	State              string                 `json:"state"`
-	Progress           float64                `json:"progress"`
-	Stage              string                 `json:"stage,omitempty"`
-	Message            string                 `json:"message,omitempty"`
-	Attempt            int32                  `json:"attempt,omitempty"`
-	CreatedAt          *time.Time             `json:"created_at,omitempty"`
-	UpdatedAt          *time.Time             `json:"updated_at,omitempty"`
-	CompletedAt        *time.Time             `json:"completed_at,omitempty"`
-	ErrorMessage       string                 `json:"error_message,omitempty"`
-	ResultSummary      string                 `json:"result_summary,omitempty"`
-	ResultOutputValues map[string]interface{} `json:"result_output_values,omitempty"`
-}
-
-type JobActivity struct {
-	JobID      string                 `json:"job_id"`
-	JobType    string                 `json:"job_type"`
-	RequestID  string                 `json:"request_id,omitempty"`
-	WorkerID   string                 `json:"worker_id,omitempty"`
-	Source     string                 `json:"source"`
-	Message    string                 `json:"message"`
-	Stage      string                 `json:"stage,omitempty"`
-	Details    map[string]interface{} `json:"details,omitempty"`
-	OccurredAt *time.Time             `json:"occurred_at,omitempty"`
-}
-
-type JobDetail struct {
-	Job         *TrackedJob   `json:"job"`
-	RunRecord   *JobRunRecord `json:"run_record,omitempty"`
-	Activities  []JobActivity `json:"activities"`
-	RelatedJobs []TrackedJob  `json:"related_jobs,omitempty"`
-	LastUpdated *time.Time    `json:"last_updated,omitempty"`
-}
-
-type SchedulerJobTypeState struct {
-	JobType                       string     `json:"job_type"`
-	Enabled                       bool       `json:"enabled"`
-	PolicyError                   string     `json:"policy_error,omitempty"`
-	DetectionInFlight             bool       `json:"detection_in_flight"`
-	NextDetectionAt               *time.Time `json:"next_detection_at,omitempty"`
-	DetectionIntervalSeconds      int32      `json:"detection_interval_seconds,omitempty"`
-	DetectionTimeoutSeconds       int32      `json:"detection_timeout_seconds,omitempty"`
-	ExecutionTimeoutSeconds       int32      `json:"execution_timeout_seconds,omitempty"`
-	MaxJobsPerDetection           int32      `json:"max_jobs_per_detection,omitempty"`
-	GlobalExecutionConcurrency    int        `json:"global_execution_concurrency,omitempty"`
-	PerWorkerExecutionConcurrency int        `json:"per_worker_execution_concurrency,omitempty"`
-	RetryLimit                    int        `json:"retry_limit,omitempty"`
-	RetryBackoffSeconds           int32      `json:"retry_backoff_seconds,omitempty"`
-	DetectorAvailable             bool       `json:"detector_available"`
-	DetectorWorkerID              string     `json:"detector_worker_id,omitempty"`
-	ExecutorWorkerCount           int        `json:"executor_worker_count"`
-}
-
-func timeToPtr(t time.Time) *time.Time {
-	if t.IsZero() {
-		return nil
-	}
-	return &t
-}

weed/admin/static/js/admin.js

@@ -129,6 +129,21 @@ function setupSubmenuBehavior() {
         }
     }
 
+    // If we're on a maintenance page, expand the maintenance submenu
+    if (currentPath.startsWith('/maintenance')) {
+        const maintenanceSubmenu = document.getElementById('maintenanceSubmenu');
+        if (maintenanceSubmenu) {
+            maintenanceSubmenu.classList.add('show');
+
+            // Update the parent toggle button state
+            const toggleButton = document.querySelector('[data-bs-target="#maintenanceSubmenu"]');
+            if (toggleButton) {
+                toggleButton.classList.remove('collapsed');
+                toggleButton.setAttribute('aria-expanded', 'true');
+            }
+        }
+    }
+
     // Prevent submenu from collapsing when clicking on submenu items
     const clusterSubmenuLinks = document.querySelectorAll('#clusterSubmenu .nav-link');
     clusterSubmenuLinks.forEach(function (link) {
@@ -146,6 +161,14 @@ function setupSubmenuBehavior() {
         });
     });
 
+    const maintenanceSubmenuLinks = document.querySelectorAll('#maintenanceSubmenu .nav-link');
+    maintenanceSubmenuLinks.forEach(function (link) {
+        link.addEventListener('click', function (e) {
+            // Don't prevent the navigation, just stop the collapse behavior
+            e.stopPropagation();
+        });
+    });
+
     // Handle the main cluster toggle
     const clusterToggle = document.querySelector('[data-bs-target="#clusterSubmenu"]');
     if (clusterToggle) {
@@ -192,6 +215,28 @@ function setupSubmenuBehavior() {
         });
     }
 
+    // Handle the main maintenance toggle
+    const maintenanceToggle = document.querySelector('[data-bs-target="#maintenanceSubmenu"]');
+    if (maintenanceToggle) {
+        maintenanceToggle.addEventListener('click', function (e) {
+            e.preventDefault();
+
+            const submenu = document.getElementById('maintenanceSubmenu');
+            const isExpanded = submenu.classList.contains('show');
+
+            if (isExpanded) {
+                // Collapse
+                submenu.classList.remove('show');
+                this.classList.add('collapsed');
+                this.setAttribute('aria-expanded', 'false');
+            } else {
+                // Expand
+                submenu.classList.add('show');
+                this.classList.remove('collapsed');
+                this.setAttribute('aria-expanded', 'true');
+            }
+        });
+    }
 }
 
 // Loading indicator functions

weed/admin/topology/capacity.go

@@ -238,7 +238,7 @@ func (at *ActiveTopology) getPlanningCapacityUnsafe(disk *activeDisk) StorageSlo
 func (at *ActiveTopology) isDiskAvailableForPlanning(disk *activeDisk, taskType TaskType) bool {
 	// Check total load including pending tasks
 	totalLoad := len(disk.pendingTasks) + len(disk.assignedTasks)
-	if MaxTotalTaskLoadPerDisk > 0 && totalLoad >= MaxTotalTaskLoadPerDisk {
+	if totalLoad >= MaxTotalTaskLoadPerDisk {
 		return false
 	}
 
@@ -299,16 +299,6 @@ func (at *ActiveTopology) getEffectiveAvailableCapacityUnsafe(disk *activeDisk)
 	}
 
 	baseAvailable := disk.DiskInfo.DiskInfo.MaxVolumeCount - disk.DiskInfo.DiskInfo.VolumeCount
-	if baseAvailable <= 0 &&
-		disk.DiskInfo.DiskInfo.MaxVolumeCount == 0 &&
-		disk.DiskInfo.DiskInfo.VolumeCount == 0 &&
-		len(disk.DiskInfo.DiskInfo.VolumeInfos) == 0 &&
-		len(disk.DiskInfo.DiskInfo.EcShardInfos) == 0 {
-		// Some empty volume servers can report max_volume_counts=0 before
-		// publishing concrete slot limits. Keep one provisional slot so EC
-		// detection still sees the disk for placement planning.
-		baseAvailable = 1
-	}
 	netImpact := at.getEffectiveCapacityUnsafe(disk)
 
 	// Calculate available volume slots (negative impact reduces availability)

weed/admin/topology/capacity_limits_test.go

@@ -1,82 +0,0 @@
-package topology
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
-)
-
-func TestGetDisksWithEffectiveCapacityNotCappedAtTenByLoad(t *testing.T) {
-	t.Parallel()
-
-	activeTopology := NewActiveTopology(0)
-	if err := activeTopology.UpdateTopology(singleDiskTopologyInfoForCapacityTest()); err != nil {
-		t.Fatalf("UpdateTopology: %v", err)
-	}
-
-	const pendingTasks = 32
-	for i := 0; i < pendingTasks; i++ {
-		taskID := fmt.Sprintf("ec-capacity-%d", i)
-		err := activeTopology.AddPendingTask(TaskSpec{
-			TaskID:     taskID,
-			TaskType:   TaskTypeErasureCoding,
-			VolumeID:   uint32(i + 1),
-			VolumeSize: 1,
-			Sources: []TaskSourceSpec{
-				{
-					ServerID:      "node-a",
-					DiskID:        0,
-					StorageImpact: &StorageSlotChange{},
-				},
-			},
-			Destinations: []TaskDestinationSpec{
-				{
-					ServerID:      "node-a",
-					DiskID:        0,
-					StorageImpact: &StorageSlotChange{},
-				},
-			},
-		})
-		if err != nil {
-			t.Fatalf("AddPendingTask(%s): %v", taskID, err)
-		}
-	}
-
-	disks := activeTopology.GetDisksWithEffectiveCapacity(TaskTypeErasureCoding, "", 1)
-	if len(disks) != 1 {
-		t.Fatalf("expected disk to remain available after %d pending tasks, got %d", pendingTasks, len(disks))
-	}
-	if disks[0].LoadCount != pendingTasks {
-		t.Fatalf("unexpected load count: got=%d want=%d", disks[0].LoadCount, pendingTasks)
-	}
-}
-
-func singleDiskTopologyInfoForCapacityTest() *master_pb.TopologyInfo {
-	return &master_pb.TopologyInfo{
-		Id: "topology-test",
-		DataCenterInfos: []*master_pb.DataCenterInfo{
-			{
-				Id: "dc1",
-				RackInfos: []*master_pb.RackInfo{
-					{
-						Id: "rack1",
-						DataNodeInfos: []*master_pb.DataNodeInfo{
-							{
-								Id: "node-a",
-								DiskInfos: map[string]*master_pb.DiskInfo{
-									"hdd": {
-										DiskId:         0,
-										Type:           "hdd",
-										VolumeCount:    0,
-										MaxVolumeCount: 1000,
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-}

weed/admin/topology/internal.go

@@ -68,7 +68,7 @@ func (at *ActiveTopology) assignTaskToDisk(task *taskState) {
 func (at *ActiveTopology) isDiskAvailable(disk *activeDisk, taskType TaskType) bool {
 	// Check if disk has too many pending and active tasks
 	activeLoad := len(disk.pendingTasks) + len(disk.assignedTasks)
-	if MaxConcurrentTasksPerDisk > 0 && activeLoad >= MaxConcurrentTasksPerDisk {
+	if activeLoad >= MaxConcurrentTasksPerDisk {
 		return false
 	}
 

weed/admin/topology/storage_slot_test.go

@@ -317,60 +317,6 @@ func TestStorageSlotChangeCapacityCalculation(t *testing.T) {
 	assert.Equal(t, int32(0), reservedShard, "Should show 0 reserved shard slots")
 }
 
-func TestGetDisksWithEffectiveCapacity_UnknownEmptyDiskFallback(t *testing.T) {
-	activeTopology := NewActiveTopology(10)
-
-	topologyInfo := &master_pb.TopologyInfo{
-		DataCenterInfos: []*master_pb.DataCenterInfo{
-			{
-				Id: "dc1",
-				RackInfos: []*master_pb.RackInfo{
-					{
-						Id: "rack1",
-						DataNodeInfos: []*master_pb.DataNodeInfo{
-							{
-								Id: "empty-node",
-								DiskInfos: map[string]*master_pb.DiskInfo{
-									"hdd": {
-										DiskId:         0,
-										Type:           "hdd",
-										VolumeCount:    0,
-										MaxVolumeCount: 0,
-									},
-								},
-							},
-							{
-								Id: "used-node",
-								DiskInfos: map[string]*master_pb.DiskInfo{
-									"hdd": {
-										DiskId:         0,
-										Type:           "hdd",
-										VolumeCount:    1,
-										MaxVolumeCount: 0,
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	err := activeTopology.UpdateTopology(topologyInfo)
-	assert.NoError(t, err)
-
-	available := activeTopology.GetDisksWithEffectiveCapacity(TaskTypeErasureCoding, "", 1)
-	assert.Len(t, available, 1, "only the empty unknown-capacity disk should be treated as provisionally available")
-	if len(available) == 1 {
-		assert.Equal(t, "empty-node", available[0].NodeID)
-		assert.Equal(t, uint32(0), available[0].DiskID)
-	}
-
-	assert.Equal(t, int64(1), activeTopology.GetEffectiveAvailableCapacity("empty-node", 0))
-	assert.Equal(t, int64(0), activeTopology.GetEffectiveAvailableCapacity("used-node", 0))
-}
-
 // TestECMultipleTargets demonstrates proper handling of EC operations with multiple targets
 func TestECMultipleTargets(t *testing.T) {
 	activeTopology := NewActiveTopology(10)

weed/admin/topology/types.go

@@ -26,17 +26,17 @@ const (
 
 // Task and capacity management configuration constants
 const (
-	// MaxConcurrentTasksPerDisk defines the maximum number of pending+assigned tasks per disk.
-	// Set to 0 to disable hard load capping and rely on effective capacity checks.
-	MaxConcurrentTasksPerDisk = 0
+	// MaxConcurrentTasksPerDisk defines the maximum number of concurrent tasks per disk
+	// This prevents overloading a single disk with too many simultaneous operations
+	MaxConcurrentTasksPerDisk = 10
 
-	// MaxTotalTaskLoadPerDisk defines the maximum total planning load (pending + active) per disk.
-	// Set to 0 to disable hard load capping for planning.
-	MaxTotalTaskLoadPerDisk = 0
+	// MaxTotalTaskLoadPerDisk defines the maximum total task load (pending + active) per disk
+	// This allows more tasks to be queued but limits the total pipeline depth
+	MaxTotalTaskLoadPerDisk = 20
 
-	// MaxTaskLoadForECPlacement defines the maximum task load to consider a disk for EC placement.
-	// Set to 0 to disable this filter.
-	MaxTaskLoadForECPlacement = 0
+	// MaxTaskLoadForECPlacement defines the maximum task load to consider a disk for EC placement
+	// This threshold ensures disks aren't overloaded when planning EC operations
+	MaxTaskLoadForECPlacement = 10
 )
 
 // StorageSlotChange represents storage impact at both volume and shard levels

weed/admin/view/app/admin_templ.go

@@ -41,7 +41,7 @@ func Admin(data dash.AdminData) templ.Component {
 		var templ_7745c5c3_Var2 string
 		templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalVolumes))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 35, Col: 73}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 35, Col: 73}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
 		if templ_7745c5c3_Err != nil {
@@ -54,7 +54,7 @@ func Admin(data dash.AdminData) templ.Component {
 		var templ_7745c5c3_Var3 string
 		templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(formatNumber(data.TotalFiles))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 55, Col: 66}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 55, Col: 66}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
 		if templ_7745c5c3_Err != nil {
@@ -67,7 +67,7 @@ func Admin(data dash.AdminData) templ.Component {
 		var templ_7745c5c3_Var4 string
 		templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(data.TotalSize))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 75, Col: 64}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 75, Col: 64}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
 		if templ_7745c5c3_Err != nil {
@@ -80,7 +80,7 @@ func Admin(data dash.AdminData) templ.Component {
 		var templ_7745c5c3_Var5 string
 		templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d MB", data.VolumeSizeLimitMB))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 95, Col: 81}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 95, Col: 81}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 		if templ_7745c5c3_Err != nil {
@@ -93,7 +93,7 @@ func Admin(data dash.AdminData) templ.Component {
 		var templ_7745c5c3_Var6 string
 		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalEcVolumes))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 118, Col: 75}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 118, Col: 75}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 		if templ_7745c5c3_Err != nil {
@@ -106,7 +106,7 @@ func Admin(data dash.AdminData) templ.Component {
 		var templ_7745c5c3_Var7 string
 		templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalEcShards))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 138, Col: 74}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 138, Col: 74}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
 		if templ_7745c5c3_Err != nil {
@@ -124,7 +124,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var8 templ.SafeURL
 			templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("http://%s/ui/index.html", master.Address)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 176, Col: 126}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 176, Col: 126}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
 			if templ_7745c5c3_Err != nil {
@@ -137,7 +137,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var9 string
 			templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(master.Address)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 177, Col: 67}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 177, Col: 67}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
 			if templ_7745c5c3_Err != nil {
@@ -170,7 +170,7 @@ func Admin(data dash.AdminData) templ.Component {
 		var templ_7745c5c3_Var10 string
 		templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.MasterNodes)))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 210, Col: 85}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 210, Col: 85}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
 		if templ_7745c5c3_Err != nil {
@@ -183,7 +183,7 @@ func Admin(data dash.AdminData) templ.Component {
 		var templ_7745c5c3_Var11 string
 		templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.VolumeServers)))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 218, Col: 87}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 218, Col: 87}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
 		if templ_7745c5c3_Err != nil {
@@ -196,7 +196,7 @@ func Admin(data dash.AdminData) templ.Component {
 		var templ_7745c5c3_Var12 string
 		templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.FilerNodes)))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 226, Col: 84}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 226, Col: 84}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
 		if templ_7745c5c3_Err != nil {
@@ -209,7 +209,7 @@ func Admin(data dash.AdminData) templ.Component {
 		var templ_7745c5c3_Var13 string
 		templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.MessageBrokers)))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 234, Col: 88}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 234, Col: 88}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
 		if templ_7745c5c3_Err != nil {
@@ -227,7 +227,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var14 string
 			templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(vs.ID)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 281, Col: 54}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 281, Col: 54}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
 			if templ_7745c5c3_Err != nil {
@@ -240,7 +240,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var15 templ.SafeURL
 			templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("http://%s/ui/index.html", vs.GetDisplayAddress())))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 283, Col: 134}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 283, Col: 134}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
 			if templ_7745c5c3_Err != nil {
@@ -253,7 +253,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var16 string
 			templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(vs.GetDisplayAddress())
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 284, Col: 75}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 284, Col: 75}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
 			if templ_7745c5c3_Err != nil {
@@ -266,7 +266,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var17 string
 			templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(vs.DataCenter)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 288, Col: 62}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 288, Col: 62}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 			if templ_7745c5c3_Err != nil {
@@ -279,7 +279,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var18 string
 			templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(vs.Rack)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 289, Col: 56}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 289, Col: 56}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
 			if templ_7745c5c3_Err != nil {
@@ -292,7 +292,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var19 string
 			templ_7745c5c3_Var19, templ_7745c5c3_Err = templruntime.SanitizeStyleAttributeValues(fmt.Sprintf("width: %d%%", calculatePercent(vs.Volumes, vs.MaxVolumes)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 293, Col: 135}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 293, Col: 135}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
 			if templ_7745c5c3_Err != nil {
@@ -305,7 +305,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var20 string
 			templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d/%d", vs.Volumes, vs.MaxVolumes))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 294, Col: 104}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 294, Col: 104}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
 			if templ_7745c5c3_Err != nil {
@@ -323,7 +323,7 @@ func Admin(data dash.AdminData) templ.Component {
 				var templ_7745c5c3_Var21 string
 				templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", vs.EcShards))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 300, Col: 127}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 300, Col: 127}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21))
 				if templ_7745c5c3_Err != nil {
@@ -341,7 +341,7 @@ func Admin(data dash.AdminData) templ.Component {
 					var templ_7745c5c3_Var22 string
 					templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d vol", vs.EcVolumes))
 					if templ_7745c5c3_Err != nil {
-						return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 302, Col: 119}
+						return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 302, Col: 119}
 					}
 					_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22))
 					if templ_7745c5c3_Err != nil {
@@ -365,7 +365,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var23 string
 			templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(vs.DiskUsage))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 308, Col: 74}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 308, Col: 74}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23))
 			if templ_7745c5c3_Err != nil {
@@ -378,7 +378,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var24 string
 			templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(vs.DiskCapacity))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 308, Col: 107}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 308, Col: 107}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var24))
 			if templ_7745c5c3_Err != nil {
@@ -407,7 +407,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var25 templ.SafeURL
 			templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("http://%s", filer.Address)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 361, Col: 111}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 361, Col: 111}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25))
 			if templ_7745c5c3_Err != nil {
@@ -420,7 +420,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var26 string
 			templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(filer.Address)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 362, Col: 66}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 362, Col: 66}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26))
 			if templ_7745c5c3_Err != nil {
@@ -433,7 +433,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var27 string
 			templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(filer.DataCenter)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 366, Col: 65}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 366, Col: 65}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27))
 			if templ_7745c5c3_Err != nil {
@@ -446,7 +446,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var28 string
 			templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(filer.Rack)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 367, Col: 59}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 367, Col: 59}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28))
 			if templ_7745c5c3_Err != nil {
@@ -459,7 +459,7 @@ func Admin(data dash.AdminData) templ.Component {
 			var templ_7745c5c3_Var29 string
 			templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(filer.LastUpdated.Format("2006-01-02 15:04:05"))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 368, Col: 96}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 368, Col: 96}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
 			if templ_7745c5c3_Err != nil {
@@ -483,7 +483,7 @@ func Admin(data dash.AdminData) templ.Component {
 		var templ_7745c5c3_Var30 string
 		templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04:05"))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 392, Col: 81}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/admin.templ`, Line: 392, Col: 81}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
 		if templ_7745c5c3_Err != nil {

weed/admin/view/app/cluster_brokers_templ.go

@@ -41,7 +41,7 @@ func ClusterBrokers(data dash.ClusterBrokersData) templ.Component {
 		var templ_7745c5c3_Var2 string
 		templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalBrokers))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_brokers.templ`, Line: 34, Col: 47}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_brokers.templ`, Line: 34, Col: 47}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
 		if templ_7745c5c3_Err != nil {
@@ -64,7 +64,7 @@ func ClusterBrokers(data dash.ClusterBrokersData) templ.Component {
 				var templ_7745c5c3_Var3 string
 				templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(broker.Address)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_brokers.templ`, Line: 70, Col: 27}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_brokers.templ`, Line: 70, Col: 27}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
 				if templ_7745c5c3_Err != nil {
@@ -77,7 +77,7 @@ func ClusterBrokers(data dash.ClusterBrokersData) templ.Component {
 				var templ_7745c5c3_Var4 string
 				templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(broker.Version)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_brokers.templ`, Line: 73, Col: 66}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_brokers.templ`, Line: 73, Col: 66}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
 				if templ_7745c5c3_Err != nil {
@@ -90,7 +90,7 @@ func ClusterBrokers(data dash.ClusterBrokersData) templ.Component {
 				var templ_7745c5c3_Var5 string
 				templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(broker.DataCenter)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_brokers.templ`, Line: 76, Col: 69}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_brokers.templ`, Line: 76, Col: 69}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 				if templ_7745c5c3_Err != nil {
@@ -103,7 +103,7 @@ func ClusterBrokers(data dash.ClusterBrokersData) templ.Component {
 				var templ_7745c5c3_Var6 string
 				templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(broker.Rack)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_brokers.templ`, Line: 79, Col: 63}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_brokers.templ`, Line: 79, Col: 63}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 				if templ_7745c5c3_Err != nil {
@@ -117,7 +117,7 @@ func ClusterBrokers(data dash.ClusterBrokersData) templ.Component {
 					var templ_7745c5c3_Var7 string
 					templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(broker.CreatedAt.Format("2006-01-02 15:04:05"))
 					if templ_7745c5c3_Err != nil {
-						return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_brokers.templ`, Line: 83, Col: 60}
+						return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_brokers.templ`, Line: 83, Col: 60}
 					}
 					_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
 					if templ_7745c5c3_Err != nil {
@@ -151,7 +151,7 @@ func ClusterBrokers(data dash.ClusterBrokersData) templ.Component {
 		var templ_7745c5c3_Var8 string
 		templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04:05"))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_brokers.templ`, Line: 108, Col: 67}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_brokers.templ`, Line: 108, Col: 67}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
 		if templ_7745c5c3_Err != nil {

weed/admin/view/app/cluster_collections_templ.go

@@ -41,7 +41,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 		var templ_7745c5c3_Var2 string
 		templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalCollections))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 34, Col: 77}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 34, Col: 77}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
 		if templ_7745c5c3_Err != nil {
@@ -54,7 +54,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 		var templ_7745c5c3_Var3 string
 		templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalVolumes))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 54, Col: 73}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 54, Col: 73}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
 		if templ_7745c5c3_Err != nil {
@@ -67,7 +67,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 		var templ_7745c5c3_Var4 string
 		templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalEcVolumes))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 74, Col: 75}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 74, Col: 75}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
 		if templ_7745c5c3_Err != nil {
@@ -80,7 +80,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 		var templ_7745c5c3_Var5 string
 		templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalFiles))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 94, Col: 71}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 94, Col: 71}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 		if templ_7745c5c3_Err != nil {
@@ -93,7 +93,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 		var templ_7745c5c3_Var6 string
 		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(data.TotalSize))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 114, Col: 64}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 114, Col: 64}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 		if templ_7745c5c3_Err != nil {
@@ -116,7 +116,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 				var templ_7745c5c3_Var7 templ.SafeURL
 				templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/storage/collections/%s", collection.Name)))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 152, Col: 123}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 152, Col: 123}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
 				if templ_7745c5c3_Err != nil {
@@ -129,7 +129,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 				var templ_7745c5c3_Var8 string
 				templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(collection.Name)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 153, Col: 68}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 153, Col: 68}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
 				if templ_7745c5c3_Err != nil {
@@ -142,7 +142,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 				var templ_7745c5c3_Var9 templ.SafeURL
 				templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/storage/volumes?collection=%s", collection.Name)))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 157, Col: 130}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 157, Col: 130}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
 				if templ_7745c5c3_Err != nil {
@@ -156,7 +156,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 					var templ_7745c5c3_Var10 string
 					templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", collection.VolumeCount))
 					if templ_7745c5c3_Err != nil {
-						return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 161, Col: 94}
+						return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 161, Col: 94}
 					}
 					_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
 					if templ_7745c5c3_Err != nil {
@@ -175,7 +175,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 				var templ_7745c5c3_Var11 templ.SafeURL
 				templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/storage/ec-shards?collection=%s", collection.Name)))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 169, Col: 132}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 169, Col: 132}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
 				if templ_7745c5c3_Err != nil {
@@ -189,7 +189,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 					var templ_7745c5c3_Var12 string
 					templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", collection.EcVolumeCount))
 					if templ_7745c5c3_Err != nil {
-						return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 173, Col: 96}
+						return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 173, Col: 96}
 					}
 					_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
 					if templ_7745c5c3_Err != nil {
@@ -208,7 +208,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 				var templ_7745c5c3_Var13 string
 				templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", collection.FileCount))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 183, Col: 88}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 183, Col: 88}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
 				if templ_7745c5c3_Err != nil {
@@ -221,7 +221,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 				var templ_7745c5c3_Var14 string
 				templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(collection.TotalSize))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 189, Col: 82}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 189, Col: 82}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
 				if templ_7745c5c3_Err != nil {
@@ -254,7 +254,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 					var templ_7745c5c3_Var16 string
 					templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(templ.CSSClasses(templ_7745c5c3_Var15).String())
 					if templ_7745c5c3_Err != nil {
-						return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 1, Col: 0}
+						return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 1, Col: 0}
 					}
 					_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
 					if templ_7745c5c3_Err != nil {
@@ -267,7 +267,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 					var templ_7745c5c3_Var17 string
 					templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(diskType)
 					if templ_7745c5c3_Err != nil {
-						return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 197, Col: 131}
+						return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 197, Col: 131}
 					}
 					_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 					if templ_7745c5c3_Err != nil {
@@ -291,7 +291,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 				var templ_7745c5c3_Var18 string
 				templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(collection.Name)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 208, Col: 78}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 208, Col: 78}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
 				if templ_7745c5c3_Err != nil {
@@ -304,7 +304,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 				var templ_7745c5c3_Var19 string
 				templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(collection.DataCenter)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 209, Col: 90}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 209, Col: 90}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
 				if templ_7745c5c3_Err != nil {
@@ -317,7 +317,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 				var templ_7745c5c3_Var20 string
 				templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", collection.VolumeCount))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 210, Col: 112}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 210, Col: 112}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
 				if templ_7745c5c3_Err != nil {
@@ -330,7 +330,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 				var templ_7745c5c3_Var21 string
 				templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", collection.EcVolumeCount))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 211, Col: 117}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 211, Col: 117}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21))
 				if templ_7745c5c3_Err != nil {
@@ -343,7 +343,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 				var templ_7745c5c3_Var22 string
 				templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", collection.FileCount))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 212, Col: 108}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 212, Col: 108}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22))
 				if templ_7745c5c3_Err != nil {
@@ -356,7 +356,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 				var templ_7745c5c3_Var23 string
 				templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", collection.TotalSize))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 213, Col: 108}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 213, Col: 108}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23))
 				if templ_7745c5c3_Err != nil {
@@ -369,7 +369,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 				var templ_7745c5c3_Var24 string
 				templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinStringErrs(formatDiskTypes(collection.DiskTypes))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 214, Col: 106}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 214, Col: 106}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var24))
 				if templ_7745c5c3_Err != nil {
@@ -397,7 +397,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component {
 		var templ_7745c5c3_Var25 string
 		templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04:05"))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 238, Col: 81}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_collections.templ`, Line: 238, Col: 81}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25))
 		if templ_7745c5c3_Err != nil {

weed/admin/view/app/cluster_ec_shards_templ.go

@@ -56,7 +56,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 				var templ_7745c5c3_Var2 string
 				templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.FilterCollection)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 22, Col: 96}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 22, Col: 96}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
 				if templ_7745c5c3_Err != nil {
@@ -119,7 +119,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 		var templ_7745c5c3_Var3 string
 		templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalShards))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 54, Col: 81}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 54, Col: 81}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
 		if templ_7745c5c3_Err != nil {
@@ -132,7 +132,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 		var templ_7745c5c3_Var4 string
 		templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalVolumes))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 69, Col: 82}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 69, Col: 82}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
 		if templ_7745c5c3_Err != nil {
@@ -145,7 +145,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 		var templ_7745c5c3_Var5 string
 		templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.VolumesWithAllShards))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 84, Col: 90}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 84, Col: 90}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
 		if templ_7745c5c3_Err != nil {
@@ -158,7 +158,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 		var templ_7745c5c3_Var6 string
 		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.VolumesWithMissingShards))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 100, Col: 94}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 100, Col: 94}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 		if templ_7745c5c3_Err != nil {
@@ -312,7 +312,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 			var templ_7745c5c3_Var7 string
 			templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", shard.VolumeID))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 203, Col: 84}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 203, Col: 84}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
 			if templ_7745c5c3_Err != nil {
@@ -335,7 +335,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 					var templ_7745c5c3_Var8 string
 					templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(shard.Collection)
 					if templ_7745c5c3_Err != nil {
-						return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 209, Col: 96}
+						return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 209, Col: 96}
 					}
 					_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
 					if templ_7745c5c3_Err != nil {
@@ -364,7 +364,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 				var templ_7745c5c3_Var9 string
 				templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(shard.DataCenter)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 220, Col: 88}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 220, Col: 88}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
 				if templ_7745c5c3_Err != nil {
@@ -382,7 +382,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 			var templ_7745c5c3_Var10 string
 			templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(shard.Server)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 224, Col: 61}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 224, Col: 61}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
 			if templ_7745c5c3_Err != nil {
@@ -400,7 +400,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 				var templ_7745c5c3_Var11 string
 				templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(shard.Rack)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 228, Col: 84}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 228, Col: 84}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
 				if templ_7745c5c3_Err != nil {
@@ -434,7 +434,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 			var templ_7745c5c3_Var12 string
 			templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", shard.VolumeID))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 241, Col: 90}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 241, Col: 90}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
 			if templ_7745c5c3_Err != nil {
@@ -452,7 +452,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 				var templ_7745c5c3_Var13 string
 				templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", shard.VolumeID))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 248, Col: 94}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 248, Col: 94}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
 				if templ_7745c5c3_Err != nil {
@@ -485,7 +485,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 				var templ_7745c5c3_Var14 string
 				templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.CurrentPage-1))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 267, Col: 129}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 267, Col: 129}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
 				if templ_7745c5c3_Err != nil {
@@ -524,7 +524,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 				var templ_7745c5c3_Var15 string
 				templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.CurrentPage-1))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 288, Col: 129}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 288, Col: 129}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
 				if templ_7745c5c3_Err != nil {
@@ -537,7 +537,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 				var templ_7745c5c3_Var16 string
 				templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.CurrentPage-1))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 288, Col: 170}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 288, Col: 170}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
 				if templ_7745c5c3_Err != nil {
@@ -555,7 +555,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 			var templ_7745c5c3_Var17 string
 			templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.CurrentPage))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 293, Col: 80}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 293, Col: 80}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 			if templ_7745c5c3_Err != nil {
@@ -573,7 +573,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 				var templ_7745c5c3_Var18 string
 				templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.CurrentPage+1))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 298, Col: 129}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 298, Col: 129}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
 				if templ_7745c5c3_Err != nil {
@@ -586,7 +586,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 				var templ_7745c5c3_Var19 string
 				templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.CurrentPage+1))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 298, Col: 170}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 298, Col: 170}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
 				if templ_7745c5c3_Err != nil {
@@ -615,7 +615,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 				var templ_7745c5c3_Var20 string
 				templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalPages))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 310, Col: 126}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 310, Col: 126}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
 				if templ_7745c5c3_Err != nil {
@@ -628,7 +628,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 				var templ_7745c5c3_Var21 string
 				templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalPages))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 310, Col: 164}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 310, Col: 164}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21))
 				if templ_7745c5c3_Err != nil {
@@ -647,7 +647,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
 				var templ_7745c5c3_Var22 string
 				templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.CurrentPage+1))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 316, Col: 129}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 316, Col: 129}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22))
 				if templ_7745c5c3_Err != nil {
@@ -700,7 +700,7 @@ func displayShardDistribution(shard dash.EcShardWithInfo, allShards []dash.EcSha
 		var templ_7745c5c3_Var24 string
 		templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinStringErrs(calculateDistributionSummary(shard.VolumeID, allShards))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 424, Col: 65}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 424, Col: 65}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var24))
 		if templ_7745c5c3_Err != nil {
@@ -750,7 +750,7 @@ func displayVolumeStatus(shard dash.EcShardWithInfo) templ.Component {
 				var templ_7745c5c3_Var26 string
 				templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(shard.MissingShards)))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 434, Col: 129}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 434, Col: 129}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26))
 				if templ_7745c5c3_Err != nil {
@@ -768,7 +768,7 @@ func displayVolumeStatus(shard dash.EcShardWithInfo) templ.Component {
 				var templ_7745c5c3_Var27 string
 				templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(shard.MissingShards)))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 436, Col: 145}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 436, Col: 145}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27))
 				if templ_7745c5c3_Err != nil {
@@ -786,7 +786,7 @@ func displayVolumeStatus(shard dash.EcShardWithInfo) templ.Component {
 				var templ_7745c5c3_Var28 string
 				templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(shard.MissingShards)))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 438, Col: 138}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 438, Col: 138}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28))
 				if templ_7745c5c3_Err != nil {
@@ -804,7 +804,7 @@ func displayVolumeStatus(shard dash.EcShardWithInfo) templ.Component {
 				var templ_7745c5c3_Var29 string
 				templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(shard.MissingShards)))
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_shards.templ`, Line: 440, Col: 137}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/cluster_ec_shards.templ`, Line: 440, Col: 137}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
 				if templ_7745c5c3_Err != nil {